gnunux/risotto
1
0
Fork 0
forked from stove/risotto
Code Issues Pull requests Projects Releases Wiki Activity

tiramisu 3 to 4

Browse source
This commit is contained in:
egarette@silique.fr 2023-06-22 16:19:44 +02:00
parent 5e735cf453
commit 186c886e84
7 changed files with 608 additions and 452 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

65
ansible/inventory.py
View file

@ -8,8 +8,8 @@ from argparse import ArgumentParser
from json import load as json_load, dumps, JSONEncoder
from os import remove
from os.path import isfile
from asyncio import run
from traceback import print_exc
from sys import stderr, argv
from risotto.machine import load, TIRAMISU_CACHE, VALUES_CACHE, INFORMATIONS_CACHE, ROUGAIL_NAMESPACE, ROUGAIL_NAMESPACE_DESCRIPTION
from tiramisu import Config
@ -40,12 +40,13 @@ class RisottoInventory(object):
parser.add_argument('--host', action='store')
parser.add_argument('--nocache', action='store_true')
parser.add_argument('--debug', action='store_true')
parser.add_argument('--pretty_print', action='store_true')
self.args = parser.parse_args()
if self.args.debug:
global DEBUG
DEBUG = True
async def run(self):
def run(self):
if self.args.list and self.args.host:
raise Exception('cannot have --list and --host together')
if self.args.list or self.args.nocache:
@ -55,20 +56,20 @@ class RisottoInventory(object):
remove(VALUES_CACHE)
if isfile(INFORMATIONS_CACHE):
remove(INFORMATIONS_CACHE)
config = await load(TIRAMISU_CACHE,
VALUES_CACHE,
INFORMATIONS_CACHE,
)
config = load(TIRAMISU_CACHE,
VALUES_CACHE,
INFORMATIONS_CACHE,
)
if self.args.list:
return await self.do_inventory(config)
return self.do_inventory(config)
elif self.args.host:
return await self.get_vars(config, self.args.host)
return self.get_vars(config, self.args.host)
raise Exception('pfff')
async def do_inventory(self,
config: Config,
) -> dict:
servers = [await subconfig.option.doc() for subconfig in await config.option.list('optiondescription') if await subconfig.information.get('module') == 'host']
def do_inventory(self,
config: Config,
) -> dict:
servers = [subconfig.doc() for subconfig in config.option.list('optiondescription') if subconfig.information.get('module') == 'host']
return dumps({
'group': {
'hosts': servers,
@ -81,44 +82,54 @@ class RisottoInventory(object):
}
})
async def get_vars(self,
config: Config,
host_name: str,
) -> dict:
def get_vars(self,
config: Config,
host_name: str,
) -> dict:
ret = {}
rougailconfig = RougailConfig.copy()
rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE
rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
for subconfig in await config.option.list('optiondescription'):
server_name = await subconfig.option.description()
module_name = await subconfig.option(await subconfig.information.get('provider:global:module_name')).value.get()
for subconfig in config.option.list('optiondescription'):
server_name = subconfig.description()
module_name = subconfig.option(subconfig.information.get('provider:global:module_name')).value.get()
if module_name == 'host' and server_name != host_name:
continue
engine = RougailSystemdTemplate(subconfig, rougailconfig)
await engine.load_variables()
engine.load_variables(with_flatten=False)
if module_name != 'host' and engine.rougail_variables_dict['general']['host'] != host_name:
continue
ret[server_name] = engine.rougail_variables_dict
ret['modules'] = await config.information.get('modules')
ret['modules'] = config.information.get('modules')
ret['delete_old_image'] = False
ret['configure_host'] = True
ret['only_machine'] = None
ret['copy_templates'] = False
ret['copy_tests'] = False
ret['host_install_dir'] = ret[host_name].pop('host_install_dir')
ret['host_install_dir'] = ret[host_name]['general']['host_install_dir']
return dumps(ret, cls=RougailEncoder)
# Get the inventory.
async def main():
def main():
try:
inv = RisottoInventory()
values = await inv.run()
print(values)
values = inv.run()
if inv.args.pretty_print:
from pprint import pprint
from json import loads
pprint(loads(values))
else:
print(values)
except Exception as err:
if DEBUG:
print_exc()
print(err)
print('---', file=stderr)
extra=''
else:
extra=f'\nmore informations with commandline "{" ".join(argv)} --debug"'
print(f'{err}{extra}', file=stderr)
exit(1)
run(main())
main()

103
sbin/risotto_display
View file

@ -1,6 +1,5 @@
#!/usr/bin/env python3
from asyncio import run
from tabulate import tabulate
from argparse import ArgumentParser
@ -18,19 +17,19 @@ def list_to_string(lst):
return lst
async def get_files_subelements(type_name, element, files_subelement, files_cols):
def get_files_subelements(type_name, element, files_subelement, files_cols):
data = {}
if not await element.option('activate').value.get():
if not element.option('activate').value.get():
return data
for subelement in files_subelement.values():
if subelement['type'] == 'subelement':
try:
value = list_to_string(await element.option(subelement['key']).value.get())
value = list_to_string(element.option(subelement['key']).value.get())
# FIXME except AttributeError:
except Exception:
value = ''
elif subelement['type'] == 'information':
value = await element.information.get(subelement['key'], '')
value = element.information.get(subelement['key'], '')
elif subelement['type'] == 'none':
value = subelement['value']
else:
@ -47,7 +46,7 @@ async def get_files_subelements(type_name, element, files_subelement, files_cols
return data
async def services(config, values):
def services(config, values):
files_subelement = {'Source': {'key': 'source', 'type': 'information'},
'Nom': {'key': 'name', 'type': 'subelement'},
'Variable': {'key': 'variable', 'type': 'subelement'},
@ -57,26 +56,26 @@ async def services(config, values):
'Moteur': {'key': 'engine', 'type': 'information'},
}
disabled_services = []
for service in await config.option.list(type="all"):
doc = await service.option.doc()
for service in config.option.list(type="all"):
doc = service.option.doc()
files_lst = []
files_cols = set()
if not await service.option('manage').value.get():
if not service.option('manage').value.get():
doc += " - unmanaged"
if not await service.option('activate').value.get():
if not service.option('activate').value.get():
disabled_services.append([doc])
else:
for type in await service.list(type="all"):
type_name = await type.option.doc()
for type in service.list(type="all"):
type_name = type.option.doc()
if type_name in ['files', 'overrides']:
for element in await type.list(type="all"):
data = await get_files_subelements(type_name, element, files_subelement, files_cols)
for element in type.list(type="all"):
data = get_files_subelements(type_name, element, files_subelement, files_cols)
if data:
files_lst.append(data)
elif type_name == 'manage':
pass
elif type_name == 'activate':
if not await type.value.get():
if not type.value.get():
doc += " - unactivated"
else:
print("FIXME " + type_name)
@ -89,19 +88,19 @@ async def services(config, values):
values["Services désactivés"] = {'keys': ['Nom'], 'lst': disabled_services}
async def table_leader(config, read_only):
def table_leader(config, read_only):
keys = ['Description']
if read_only:
keys.append('Cachée')
leadership_lst = await config.list(type="all")
leadership_lst = config.list(type="all")
leader = leadership_lst.pop(0)
leader_owner = await leader.owner.get()
follower_names = [await follower.option.name() for follower in leadership_lst]
doc = await leader.option.doc()
properties = await leader.property.get()
leader_owner = leader.owner.get()
follower_names = [follower.option.name() for follower in leadership_lst]
doc = leader.option.doc()
properties = leader.property.get()
if 'mandatory' in properties:
doc += '*'
name = await leader.option.name()
name = leader.option.name()
lst = [[f'{doc} ({name})']]
if read_only:
if 'hidden' in properties:
@ -109,7 +108,7 @@ async def table_leader(config, read_only):
else:
hidden = ''
lst[0].append(hidden)
for idx, leader_value in enumerate(await leader.value.get()):
for idx, leader_value in enumerate(leader.value.get()):
keys.append(f'Valeur {idx}')
keys.append(f'Utilisateur {idx}')
lst[0].append(leader_value)
@ -117,11 +116,11 @@ async def table_leader(config, read_only):
for follower_idx, follower_name in enumerate(follower_names):
follower_option = config.option(follower_name, idx)
if idx == 0:
doc = await follower_option.option.doc()
properties = await follower_option.property.get()
doc = follower_option.option.doc()
properties = follower_option.property.get()
if 'mandatory' in properties:
doc += '*'
name = await follower_option.option.name()
name = follower_option.option.name()
lst.append([f'{doc} ({name})'])
if read_only:
if 'hidden' in properties:
@ -130,48 +129,48 @@ async def table_leader(config, read_only):
hidden = ''
lst[-1].append(hidden)
try:
lst[follower_idx + 1].append(list_to_string(await follower_option.value.get()))
lst[follower_idx + 1].append(await follower_option.owner.get())
lst[follower_idx + 1].append(list_to_string(follower_option.value.get()))
lst[follower_idx + 1].append(follower_option.owner.get())
except PropertiesOptionError:
pass
# leader = next leader_iter
# if master_values is None:
# master_values = await subconfig.value.get()
# master_values = subconfig.value.get()
return {'keys': keys, 'lst': lst}
async def table(config, prefix_len, values, read_only):
def table(config, prefix_len, values, read_only):
lst = []
for subconfig in await config.option.list(type="all"):
for subconfig in config.option.list(type="all"):
# prefix = prefix_len * 2 * ' '
# if await subconfig.option.isoptiondescription():
# if subconfig.option.isoptiondescription():
# prefix += '=>'
# else:
# prefix += '-'
# display_str = f'{prefix} {description}'
# if name != description:
# display_str = f'{display_str} ({name})'
name = await subconfig.option.name()
doc = await subconfig.option.doc()
name = subconfig.option.name()
doc = subconfig.option.doc()
if prefix_len == 0 and ROUGAIL_NAMESPACE != name:
doc = doc.capitalize()
if prefix_len == 0 and name == 'services':
values['Services'] = {}
await services(subconfig, values['Services'])
elif await subconfig.option.isoptiondescription():
od_name = f'{doc} ({(await subconfig.option.path()).split(".", 1)[1]})'
services(subconfig, values['Services'])
elif subconfig.option.isoptiondescription():
od_name = f'{doc} ({(subconfig.option.path()).split(".", 1)[1]})'
values[od_name] = None
if await subconfig.option.isleadership():
values[od_name] = await table_leader(subconfig, read_only)
if subconfig.option.isleadership():
values[od_name] = table_leader(subconfig, read_only)
else:
values[od_name] = await table(subconfig, prefix_len + 1, values, read_only)
values[od_name] = table(subconfig, prefix_len + 1, values, read_only)
else:
value = list_to_string(await subconfig.value.get())
doc = await subconfig.option.doc()
properties = await subconfig.property.get()
value = list_to_string(subconfig.value.get())
doc = subconfig.option.doc()
properties = subconfig.property.get()
if 'mandatory' in properties:
doc += '*'
name = await subconfig.option.name()
name = subconfig.option.name()
lst.append([f'{doc} ({name})', value])
if read_only:
if 'hidden' in properties:
@ -179,7 +178,7 @@ async def table(config, prefix_len, values, read_only):
else:
hidden = ''
lst[-1].append(hidden)
lst[-1].append(await subconfig.owner.get())
lst[-1].append(subconfig.owner.get())
keys = ['Description', 'Valeur']
if read_only:
keys.append('Cachée')
@ -187,7 +186,7 @@ async def table(config, prefix_len, values, read_only):
return {'keys': keys, 'lst': lst}
async def main():
def main():
parser = ArgumentParser()
parser.add_argument('server_name')
parser.add_argument('--read_only', action='store_true')
@ -199,18 +198,18 @@ async def main():
values = {}
server_name = args.server_name
config = await load(hide_secret=HIDE_SECRET,
config = load(hide_secret=HIDE_SECRET,
original_display_name=True,
valid_mandatories=args.read_only,
)
if not args.read_only:
await config.property.read_write()
config.property.read_write()
root_option = config.option(normalize_family(server_name))
try:
await root_option.option.get()
root_option.option.get()
except AttributeError:
exit(f'Unable to find {server_name} configuration: {[await o.option.description() for o in await config.option.list(type="optiondescription")]}')
await table(root_option, 0, values, args.read_only)
exit(f'Unable to find {server_name} configuration: {[o.option.description() for o in config.option.list(type="optiondescription")]}')
table(root_option, 0, values, args.read_only)
for title, dico in values.items():
if title == 'Services':
if not dico:
@ -233,4 +232,4 @@ async def main():
print(tabulate(dico['lst'], headers=dico['keys'], tablefmt="fancy_grid"))
run(main())
main()

17
sbin/risotto_templates
View file

@ -1,13 +1,12 @@
#!/usr/bin/env python3
from asyncio import run
from argparse import ArgumentParser
from traceback import print_exc
from risotto.machine import remove_cache, build_files, INSTALL_DIR
async def main():
def main():
parser = ArgumentParser()
parser.add_argument('server_name', nargs='?')
parser.add_argument('--nocache', action='store_true')
@ -19,12 +18,12 @@ async def main():
remove_cache()
try:
await build_files(None,
args.server_name,
False,
args.copy_tests,
template=args.template,
)
build_files(None,
args.server_name,
False,
args.copy_tests,
template=args.template,
)
except Exception as err:
if args.debug:
print_exc()
@ -32,4 +31,4 @@ async def main():
print(f'templates generated in "{INSTALL_DIR}" directory')
run(main())
main()

68
src/risotto/image.py
View file

@ -19,8 +19,8 @@ class ModuleCfg():
self.depends = []
self.manuals = []
self.tests = []
self.providers = []
self.suppliers = []
#self.providers = []
#self.suppliers = []
def __repr__(self):
return str(vars(self))
@ -131,10 +131,10 @@ class Modules:
if appname not in cfg.providers[provider]:
cfg.providers[provider].append(appname)
supplier = app.get('supplier')
if supplier:
self.suppliers.setdefault(supplier, [])
if appname not in self.suppliers[supplier]:
self.suppliers[supplier].append(appname)
#if supplier:
# self.suppliers.setdefault(supplier, [])
# if appname not in self.suppliers[supplier]:
# self.suppliers[supplier].append(appname)
if 'distribution' in app and app['distribution']:
distribution = appname
else:
@ -216,33 +216,57 @@ def applicationservice_copy(src_file: str,
copytree(src_file, dst_file)
async def valid_mandatories(config):
mandatories = await config.value.mandatory()
await config.property.pop('mandatory')
def valid_mandatories(config):
mandatories = config.value.mandatory()
config.property.remove('mandatory')
hidden = {}
variables = {}
title = None
if mandatories:
server_name = None
for mandatory in mandatories:
path_server_name, path = mandatory.split('.', 1)
var_server_name = await config.option(path_server_name).option.description()
for mandatory_option in mandatories:
path_server_name, path = mandatory_option.path().split('.', 1)
var_server_name = config.option(path_server_name).description()
if server_name != var_server_name:
server_name = var_server_name
title = f'=== Missing variables for {server_name} ==='
suboption = config.option(mandatory)
text = await suboption.option.doc()
text = mandatory_option.doc()
msg = f' - {text} ({path})'
supplier = await suboption.information.get('supplier', None)
supplier = mandatory_option.information.get('supplier', None)
if supplier:
msg += f' you could add a service that provides "{supplier}"'
try:
await config.option(mandatory).value.get()
variables.setdefault(title, []).append(msg)
except PropertiesOptionError as err:
if 'hidden' not in err.proptype:
raise PropertiesOptionError(err)
hidden.setdefault(title, []).append(msg)
if mandatory_option.isfollower():
leader = mandatory_option.leader()
try:
leader_value = leader.value.get()
except PropertiesOptionError as err:
if 'hidden' not in err.proptype:
raise err from err
hidden.setdefault(title, []).append(msg)
else:
config.property.add('mandatory')
for idx in range(mandatory_option.value.len()):
try:
config.option(mandatory_option.path(), idx).value.get()
except PropertiesOptionError as err:
path = leader.path()
spath = path.split('.', 1)[1]
submsg = f'{msg} at index {idx} (value of leader "{leader.doc()}" ({spath}) is "{leader_value[idx]}")'
if 'hidden' in err.proptype:
hidden.setdefault(title, []).append(submsg)
elif 'mandatory' in err.proptype:
variables.setdefault(title, []).append(submsg)
else:
raise err from err
config.property.remove('mandatory')
else:
try:
mandatory_option.value.get()
variables.setdefault(title, []).append(msg)
except PropertiesOptionError as err:
if 'hidden' not in err.proptype:
raise err from err
hidden.setdefault(title, []).append(msg)
if not variables:
variables = hidden
return variables

310
src/risotto/machine.py
View file

@ -5,11 +5,11 @@ from .rougail.annotator import calc_providers, calc_providers_global, calc_provi
from rougail import RougailConfig, RougailConvert
from os import remove, makedirs, listdir, chmod
from os.path import isfile, isdir, abspath, join, dirname
from json import dump as json_dump, load as json_load
from pickle import dump as pickle_dump, load as pickle_load
from yaml import load as yaml_load, SafeLoader
from ipaddress import ip_network
from ipaddress import IPv4Interface, ip_network
#
from tiramisu import Config, valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal, calc_value
from tiramisu import Config, valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal, calc_value, calc_value_property_help
from rougail.utils import normalize_family
from rougail import RougailSystemdTemplate
from shutil import copy2, copytree, rmtree
@ -29,8 +29,8 @@ def tiramisu_display_name(kls,
CONFIG_FILE = 'servers.yml'
TIRAMISU_CACHE = 'tiramisu_cache.py'
VALUES_CACHE = 'values_cache.json'
INFORMATIONS_CACHE = 'informations_cache.json'
VALUES_CACHE = 'values_cache.pickle'
INFORMATIONS_CACHE = 'informations_cache.pickle'
INSTALL_DIR = RISOTTO_CONFIG['directories']['dest']
INSTALL_CONFIG_DIR = 'configurations'
INSTALL_TMPL_DIR= 'templates'
@ -47,6 +47,7 @@ FUNCTIONS = {'calc_providers': calc_providers,
'valid_in_network': valid_in_network,
'valid_not_equal': valid_not_equal,
'calc_value': calc_value,
'calc_value_property_help': calc_value_property_help,
'normalize_family': normalize_family,
}
@ -88,28 +89,28 @@ def remove_cache():
remove(INFORMATIONS_CACHE)
async def templates(server_name,
config,
just_copy=False,
copy_manuals=False,
template=None,
extra_variables=None,
):
def templates(server_name,
config,
just_copy=False,
copy_manuals=False,
template=None,
extra_variables=None,
):
subconfig = config.option(normalize_family(server_name))
try:
await subconfig.option.get()
subconfig.get()
except:
servers = [await server.option.description() for server in await config.option.list('optiondescription')]
servers = [server.description() for server in config.list('optiondescription')]
raise Exception(f'cannot find server name "{server_name}": {servers}')
rougailconfig = RougailConfig.copy()
rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE
rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
rougailconfig['tmp_dir'] = 'tmp'
rougailconfig['templates_dir'] = await subconfig.information.get('templates_dir')
rougailconfig['patches_dir'] = await subconfig.information.get('patches_dir')
rougailconfig['functions_file'] = await subconfig.information.get('functions_files')
module = await subconfig.information.get('module')
rougailconfig['templates_dir'] = subconfig.information.get('templates_dir')
rougailconfig['patches_dir'] = subconfig.information.get('patches_dir')
rougailconfig['functions_file'] = subconfig.information.get('functions_files')
module = subconfig.information.get('module')
is_host = module == 'host'
if is_host:
rougailconfig['systemd_tmpfile_delete_before_create'] = True
@ -138,15 +139,15 @@ async def templates(server_name,
engine.engines[eng] = engine.engines['none']
try:
if not template:
await engine.instance_files(extra_variables=extra_variables)
engine.instance_files(extra_variables=extra_variables)
else:
await engine.instance_file(template, extra_variables=extra_variables)
engine.instance_file(template, extra_variables=extra_variables)
except Exception as err:
print()
print(f'=== Configuration: {server_name} ===')
try:
values = await subconfig.value.dict()
await value_pprint(values, subconfig)
values = subconfig.value.dict()
value_pprint(values, subconfig)
except:
pass
raise err from err
@ -159,17 +160,17 @@ async def templates(server_name,
if copy_manuals and not is_host:
dest_dir = join(INSTALL_DIR, INSTALL_IMAGES_DIR, module)
if not isdir(dest_dir):
for manual in await subconfig.information.get('manuals_dirs'):
for manual in subconfig.information.get('manuals_dirs'):
for filename in listdir(manual):
src_file = join(manual, filename)
dst_file = join(dest_dir, filename)
copy(src_file, dst_file)
copy_tests = await config.information.get('copy_tests')
copy_tests = config.information.get('copy_tests')
if copy_tests and not is_host:
dest_dir = join(INSTALL_DIR, INSTALL_TESTS_DIR, module)
if not isdir(dest_dir):
for tests in await subconfig.information.get('tests_dirs'):
for tests in subconfig.information.get('tests_dirs'):
for filename in listdir(tests):
src_file = join(tests, filename)
dst_file = join(dest_dir, filename)
@ -178,7 +179,6 @@ async def templates(server_name,
class Loader:
def __init__(self,
clean_directories,
hide_secret,
original_display_name,
valid_mandatories,
@ -188,10 +188,6 @@ class Loader:
self.original_display_name = original_display_name
self.valid_mandatories = valid_mandatories
self.config_file = config_file
if clean_directories:
if isdir(INSTALL_DIR):
rmtree(INSTALL_DIR)
makedirs(INSTALL_DIR)
def load_tiramisu_file(self):
"""Load config file (servers.yml) and build tiramisu file with dataset informations
@ -224,28 +220,54 @@ class Loader:
rougail = RougailConvert(cfg)
for host_name, datas in self.servers_json['hosts'].items():
for server_name, server_datas in datas['servers'].items():
for zone in server_datas['informations']['zones_name']:
if 'provider_zone' not in server_datas and 'zones_name' not in server_datas:
raise Exception(f'cannot find "zones_name" attribute for server "{server_name}"')
if 'provider_zone' in server_datas:
zones_name.setdefault(server_datas['provider_zone'], []).append(server_name)
if 'zones_name' not in server_datas:
server_datas['zones_name'] = []
if server_datas['provider_zone'] in server_datas['zones_name']:
raise Exception(_('provider_zone "{server_datas["provider_zone"]}" must not be in "zones" "{server_datas["zones_name"]}"'))
# external zone is better in first place
if server_datas['zones_name'] and self.servers_json['zones']['external_zone'] == server_datas['zones_name'][0]:
server_datas['zones_name'].append(server_datas['provider_zone'])
else:
server_datas['zones_name'].insert(0, server_datas['provider_zone'])
# if server_datas['zones_name'] and server_datas['provider_zone'] == self.servers_json['zones']['external_zone']:
# server_datas['zones_name'].insert(0, server_datas['provider_zone'])
# else:
# server_datas['zones_name'].append(server_datas['provider_zone'])
for zone in server_datas['zones_name']:
zones_name.setdefault(zone, []).append(server_name)
self.zones = {}
zones_network = ip_network(self.servers_json['zones']['network'])
zone_start_ip = zones_network.network_address
domain_name = self.servers_json['zones']['prefix_domain_name']
for idx, zone_name in enumerate(zones_name):
sub_network = ip_network(f'{zone_start_ip}/28')
if not sub_network.subnet_of(zones_network):
raise Exception('not enough IP available')
if sub_network.num_addresses < len(zones_name[zone_name]):
#FIXME should try to increase network!
raise Exception(f'network too small for zone {zone_name}')
if idx == 0:
for zone_name in zones_name:
len_zone = len(zones_name[zone_name])
for zone_cidr in [29, 28, 27, 26]:
try:
sub_network = ip_network(f'{zone_start_ip}/{zone_cidr}')
except ValueError:
# calc network address for this mask
zone_start_ip = IPv4Interface(f'{zone_start_ip}/{zone_cidr}').network.broadcast_address + 1
sub_network = ip_network(f'{zone_start_ip}/{zone_cidr}')
if not sub_network.subnet_of(zones_network):
raise Exception('not enough IP available')
length = sub_network.num_addresses - 3 # network + broadcast + host
if length >= len_zone:
break
else:
raise Exception(f'network too small for zone "{zone_name}" ({sub_network.num_addresses - 2} < {len_zone})')
if self.servers_json['zones']['external_zone'] == zone_name:
zone_domaine_name = domain_name
else:
zone_domaine_name = zone_name + '.' + domain_name
network = sub_network.network_address
self.zones[zone_name] = {'domain_name': zone_domaine_name,
'network': str(sub_network),
'host_ip': str(network + 1),
'length': length,
'start_ip': str(network + 2)
}
zone_start_ip = str(sub_network.broadcast_address + 1)
@ -266,9 +288,13 @@ class Loader:
# load host
module_info = modules.get('host')
tls_host_name = f'{server_name}.{self.zones[list(self.zones)[0]]["domain_name"]}'
cfg['risotto_globals'][host_name] = {'global:server_name': host_name,
'global:server_names': [host_name for zone in self.zones],
'global:zones_name': list(self.zones),
'global:module_name': 'host',
'global:host_install_dir': abspath(INSTALL_DIR),
'global:tls_server': tls_host_name,
}
functions_files |= set(module_info.functions_file)
self.load_dictionaries(cfg,
@ -280,7 +306,7 @@ class Loader:
modules_info = {}
for server_name, server_datas in datas['servers'].items():
module_info = modules.get(server_datas['applicationservice'])
zones_name = server_datas['informations']['zones_name']
zones_name = server_datas['zones_name']
values = [f'{server_name}.{self.zones[zone_name]["domain_name"]}' for zone_name in zones_name]
if server_datas['applicationservice'] == 'tls':
true_host_name = f'{server_name}.{self.zones[list(self.zones)[0]]["domain_name"]}'
@ -292,7 +318,10 @@ class Loader:
'global:zones_name': zones_name,
'global:zones_list': list(range(len(zones_name))),
'global:module_name': server_datas['applicationservice'],
'global:prefix_domain_name': self.servers_json['zones']['prefix_domain_name']
}
if 'provider_zone' in server_datas:
cfg['risotto_globals'][true_host_name]['global:provider_zone'] = server_datas['provider_zone']
server_datas['server_name'] = true_host_name
functions_files |= set(module_info.functions_file)
self.load_dictionaries(cfg,
@ -309,26 +338,24 @@ class Loader:
zones = set()
dns_module_name = None
for host in self.servers_json['hosts'].values():
zones = [None, None]
zones = [self.servers_json['zones']['external_zone'], None]
for server_name, datas in host['servers'].items():
if not 'applicationservice' in datas:
raise Exception(f'cannot find applicationservice for "{server_name}"')
if datas['applicationservice'] == 'tls':
raise Exception(f'forbidden module name "tls" for server {server_name}')
raise Exception(f'forbidden module name "tls" for server "{server_name}"')
#FIXME use provider!
if datas['applicationservice'] == 'nginx-reverse-proxy' and len(datas['informations']['zones_name']) > 0:
if datas['applicationservice'] == 'nginx-reverse-proxy' and len(datas['zones_name']) > 0:
if dns_module_name:
break
# always add tls machine in second zone of reverse proxy
zones[1] = datas['informations']['zones_name'][0]
if datas['applicationservice'] == 'unbound':
# always add tls machine in second zone of reverse proxy
zones[0] = datas['informations']['zones_name'][0]
zones[1] = datas['provider_zone']
if None in zones:
zones = []
else:
if zones[0] == zones[1]:
zones = [zones[0]]
host['servers']['tls'] = {'applicationservice': 'tls',
'informations': {'zones_name': list(zones)},
'zones_name': list(zones),
}
def load_dictionaries(self, cfg, module_info, server_name, rougail):
@ -344,7 +371,7 @@ class Loader:
self.manuals_dirs[server_name] = module_info.manuals
self.tests_dirs[server_name] = module_info.tests
async def tiramisu_file_to_tiramisu(self):
def tiramisu_file_to_tiramisu(self):
# l
tiramisu_space = FUNCTIONS.copy()
try:
@ -356,78 +383,78 @@ class Loader:
display_name = None
else:
display_name = tiramisu_display_name
self.config = await Config(tiramisu_space['option_0'],
self.config = Config(tiramisu_space['option_0'],
display_name=display_name,
)
async def load_values_and_informations(self):
def load_values_and_informations(self):
config = self.config
await config.property.read_write()
await config.property.pop('validator')
await config.property.pop('cache')
config.property.read_write()
config.property.remove('validator')
config.property.remove('cache')
load_zones(self.zones, self.servers_json['hosts'])
await config.information.set('zones', self.zones)
config.information.set('zones', self.zones)
for host_name, hosts_datas in self.servers_json['hosts'].items():
information = config.option(normalize_family(host_name)).information
await information.set('module', 'host')
await information.set('templates_dir', self.templates_dir[host_name])
await information.set('patches_dir', self.patches_dir[host_name])
await information.set('functions_files', self.functions_files[host_name])
await self.set_values(host_name, config, hosts_datas)
information.set('module', 'host')
information.set('templates_dir', self.templates_dir[host_name])
information.set('patches_dir', self.patches_dir[host_name])
information.set('functions_files', self.functions_files[host_name])
self.set_values(host_name, config, hosts_datas)
for datas in hosts_datas['servers'].values():
server_name = datas['server_name']
information = config.option(normalize_family(server_name)).information
await information.set('module', datas['applicationservice'])
await information.set('templates_dir', self.templates_dir[server_name])
await information.set('patches_dir', self.patches_dir[server_name])
await information.set('functions_files', self.functions_files[server_name])
await information.set('manuals_dirs', self.manuals_dirs[server_name])
await information.set('tests_dirs', self.tests_dirs[server_name])
await self.set_values(server_name, config, datas)
information.set('module', datas['applicationservice'])
information.set('templates_dir', self.templates_dir[server_name])
information.set('patches_dir', self.patches_dir[server_name])
information.set('functions_files', self.functions_files[server_name])
information.set('manuals_dirs', self.manuals_dirs[server_name])
information.set('tests_dirs', self.tests_dirs[server_name])
self.set_values(server_name, config, datas)
await config.information.set('copy_tests', False)
config.information.set('copy_tests', False)
# FIXME only one host_name is supported
await config.information.set('modules', self.modules[host_name])
# await config.information.set('modules', {module_name: module_info.depends for module_name, module_info in self.module_infos.items() if module_name in modules})
await config.property.add('cache')
config.information.set('modules', self.modules[host_name])
# config.information.set('modules', {module_name: module_info.depends for module_name, module_info in self.module_infos.items() if module_name in modules})
config.property.add('cache')
if self.valid_mandatories:
messages = await valid_mandatories(config)
messages = valid_mandatories(config)
if messages:
msg = ''
for title, variables in messages.items():
msg += '\n' + title + '\n'
msg += '\n'.join(variables)
raise Exception(msg)
await config.property.read_only()
with open(VALUES_CACHE, 'w') as fh:
json_dump(await config.value.exportation(), fh)
with open(INFORMATIONS_CACHE, 'w') as fh:
json_dump(await config.information.exportation(), fh)
config.property.read_only()
with open(VALUES_CACHE, 'wb') as fh:
pickle_dump(config.value.exportation(), fh)
with open(INFORMATIONS_CACHE, 'wb') as fh:
pickle_dump(config.information.exportation(), fh)
async def set_values(self,
server_name,
config,
datas,
):
def set_values(self,
server_name,
config,
datas,
):
if 'values' not in datas:
return
if not isinstance(datas['values'], dict):
raise Exception(f'Values of "{server_name}" are not a dict: {datas["values"]}')
server_path = normalize_family(server_name)
await config.owner.set(self.config_file)
config.owner.set(self.config_file)
for vpath, value in datas['values'].items():
path = f'{server_path}.{vpath}'
try:
if isinstance(value, dict):
for idx, val in value.items():
await config.option(path, int(idx)).value.set(val)
config.option(path, int(idx)).value.set(val)
else:
await config.option(path).value.set(value)
config.option(path).value.set(value)
except Exception as err:
await value_pprint(await config.value.dict(), config)
value_pprint(config.value.dict(), config)
error_msg = f'cannot configure variable {vpath} for server "{server_name}": {err}'
raise Exception(error_msg) from err
await config.owner.set('user')
config.owner.set('user')
class LoaderCache(Loader):
@ -435,63 +462,60 @@ class LoaderCache(Loader):
with open(TIRAMISU_CACHE) as fh:
self.tiram_obj = fh.read()
async def load_values_and_informations(self):
with open(VALUES_CACHE, 'r') as fh:
await self.config.value.importation(json_load(fh))
with open(INFORMATIONS_CACHE, 'r') as fh:
informations = json_load(fh)
# null is not a valid key in json => 'null'
informations[None] = informations.pop('null')
await self.config.information.importation(informations)
def load_values_and_informations(self):
with open(VALUES_CACHE, 'rb') as fh:
self.config.value.importation(pickle_load(fh))
with open(INFORMATIONS_CACHE, 'rb') as fh:
self.config.information.importation(pickle_load(fh))
async def load(clean_directories=False,
hide_secret=False,
original_display_name: bool=False,
valid_mandatories: bool=True,
copy_tests: bool=False,
):
def load(hide_secret=False,
original_display_name: bool=False,
valid_mandatories: bool=True,
copy_tests: bool=False,
):
if isfile(TIRAMISU_CACHE) and isfile(VALUES_CACHE) and isfile(INFORMATIONS_CACHE):
loader_obj = LoaderCache
else:
loader_obj = Loader
loader = loader_obj(clean_directories,
hide_secret,
loader = loader_obj(hide_secret,
original_display_name,
valid_mandatories,
)
loader.load_tiramisu_file()
await loader.tiramisu_file_to_tiramisu()
await loader.load_values_and_informations()
loader.tiramisu_file_to_tiramisu()
loader.load_values_and_informations()
config = loader.config
await config.property.read_only()
await config.information.set('copy_tests', copy_tests)
await config.cache.reset()
config.property.read_only()
config.information.set('copy_tests', copy_tests)
config.cache.reset()
return config
async def build_files(hostname: str,
only_machine: str,
just_copy: bool,
copy_tests: bool,
template: str=None,
) -> None:
def build_files(hostname: str,
only_machine: str,
just_copy: bool,
copy_tests: bool,
template: str=None,
) -> None:
if isdir(INSTALL_DIR):
rmtree(INSTALL_DIR)
makedirs(INSTALL_DIR)
with open(CONFIG_FILE, 'r') as server_fh:
servers_json = yaml_load(server_fh, Loader=SafeLoader)
config = await load(copy_tests=copy_tests)
machines = [await subconfig.option.description() for subconfig in await config.option.list(type='optiondescription')]
config = load(copy_tests=copy_tests)
machines = [subconfig.description() for subconfig in config.option.list(type='optiondescription')]
certificates = {'certificates': {},
'configuration': servers_json['certificates'],
}
# get certificates informations
tls_machine = None
tls_machine = config.option(f'{normalize_family(hostname)}.general.tls_server').value.get()
for machine in machines:
if machine.startswith('tls.'):
tls_machine = machine
if machine == tls_machine:
continue
if hostname is None:
# FIXME multi host!
hostname = await config.option(normalize_family(machine)).option('general.host_name').value.get()
hostname = config.option(normalize_family(machine)).option('general.host_name').value.get()
if just_copy:
continue
is_host = machine == hostname
@ -500,24 +524,24 @@ async def build_files(hostname: str,
machine_config = config.option(normalize_family(machine))
certificate_names = []
private_names = []
for service in await machine_config.option('services').option.list('optiondescription'):
if not await service.option('activate').value.get():
for service in machine_config.option('services').list('optiondescription'):
if not service.option('activate').value.get():
continue
# if await service.option('manage').value.get():
# if service.option('manage').value.get():
# certificate_type = 'server'
# else:
# certificate_type = 'client'
tls_ca_directory = await machine_config.option('general.tls_ca_directory').value.get()
tls_cert_directory = await machine_config.option('general.tls_cert_directory').value.get()
tls_key_directory = await machine_config.option('general.tls_key_directory').value.get()
tls_ca_directory = machine_config.option('general.tls_ca_directory').value.get()
tls_cert_directory = machine_config.option('general.tls_cert_directory').value.get()
tls_key_directory = machine_config.option('general.tls_key_directory').value.get()
try:
for certificate in await service.option('certificates').option.list('all'):
if not await certificate.option('activate').value.get():
for certificate in service.option('certificates').list('all'):
if not certificate.option('activate').value.get():
continue
certificate_data = await certificate.value.dict()
certificate_data['type'] = await certificate.information.get('type')
certificate_data['authority'] = join(tls_ca_directory, await certificate.information.get('authority') + '.crt')
certificate_data['format'] = await certificate.information.get('format')
certificate_data = {key.rsplit('.', 1)[1]: value for key, value in certificate.value.dict().items()}
certificate_data['type'] = certificate.information.get('type')
certificate_data['authority'] = join(tls_ca_directory, certificate.information.get('authority') + '.crt')
certificate_data['format'] = certificate.information.get('format')
is_list_name = isinstance(certificate_data['name'], list)
is_list_domain = isinstance(certificate_data['domain'], list)
if is_list_name != is_list_domain:
@ -526,7 +550,7 @@ async def build_files(hostname: str,
certificate_data['provider'] = 'autosigne'
if is_list_name:
if len(certificate_data['name']) != len(certificate_data['domain']):
raise Exception('certificate name and domain name must have same lenght')
raise Exception('certificate name and domain name must have same length')
for idx, certificate_name in enumerate(certificate_data['name']):
cert_data = certificate_data.copy()
if certificate_data['format'] == 'cert_key':
@ -565,19 +589,19 @@ async def build_files(hostname: str,
continue
if only_machine and only_machine != machine:
continue
await templates(machine,
config,
just_copy=just_copy,
copy_manuals=True,
template=template,
extra_variables=certificates,
)
templates(machine,
config,
just_copy=just_copy,
copy_manuals=True,
template=template,
extra_variables=certificates,
)
is_host = machine == hostname
if is_host:
directories[machine] = '/usr/local/lib'
elif not just_copy:
machine_config = config.option(normalize_family(machine))
directories[machine] = await machine_config.option('general.config_dir').value.get()
directories[machine] = machine_config.option('general.config_dir').value.get()
if only_machine:
return directories
if only_machine:

433
src/risotto/rougail/annotator.py
View file

@ -26,19 +26,19 @@ def _parse_kwargs(provider, dns, kwargs, index=None):
continue
elif data['dns'] not in dns:
continue
del data['dns']
# del data['dns']
yield data
@multi_function
def calc_providers_global(provider, multi, value, suffix=None):
def calc_providers_global(provider, multi, unique, value, suffix=None):
if suffix is not None:
return value[int(suffix)]
return value
@multi_function
def calc_providers_follower(provider, multi, dns, leader, index, **kwargs):
def calc_providers_follower(provider, multi, unique, dns, leader, index, **kwargs):
ret = []
for data in _parse_kwargs(provider, dns, kwargs):
if 'value' not in data:
@ -64,7 +64,7 @@ def calc_providers_follower(provider, multi, dns, leader, index, **kwargs):
@multi_function
def calc_providers_dynamic_follower(provider, multi, dns, leader, index, suffix, **kwargs):
def calc_providers_dynamic_follower(provider, multi, unique, dns, leader, index, suffix, **kwargs):
ret = []
for data in _parse_kwargs(provider, dns, kwargs):
if 'value' not in data:
@ -92,7 +92,7 @@ def calc_providers_dynamic_follower(provider, multi, dns, leader, index, suffix,
@multi_function
def calc_providers_dynamic(provider, multi, dns, suffix, **kwargs):
def calc_providers_dynamic(provider, multi, unique, dns, suffix, **kwargs):
ret = []
for data in _parse_kwargs(provider, dns, kwargs):
if 'value' not in data:
@ -101,7 +101,7 @@ def calc_providers_dynamic(provider, multi, dns, suffix, **kwargs):
continue
if isinstance(data['value'], list):
for v in data['value']:
if v not in ret:
if not unique or v not in ret:
ret.append(v)
elif data['value'] not in ret:
ret.append(data['value'])
@ -112,10 +112,14 @@ def calc_providers_dynamic(provider, multi, dns, suffix, **kwargs):
@multi_function
def calc_providers(provider, multi, dns, suffix=None, **kwargs):
def calc_providers(provider, multi, unique, dns, **kwargs):
ret = []
for data in _parse_kwargs(provider, dns, kwargs):
if isinstance(data['value'], list):
commun_dns = list(set(data['dns']) & set(dns))
if len(commun_dns) == 1:
ret.append(data['value'][data['dns'].index(commun_dns[0])])
continue
for v in data['value']:
if v in ret:
continue
@ -135,216 +139,289 @@ class Annotator(Walk):
objectspace: 'RougailObjSpace',
*args):
self.objectspace = objectspace
self.set_suppliers()
self.get_suppliers_providers()
self.dispatch_provider_supplier_to_zones()
self.dispatch_provider_to_zones()
self.convert_providers()
self.convert_suppliers()
def set_suppliers(self) -> dict:
def get_suppliers_providers(self) -> None:
""" get supplier informations
return something like:
{'Host': ['host1.example.net', 'host2.example.net']}
"""
self.suppliers = {}
for variable in self.get_variables():
if not hasattr(variable, 'supplier') or ':' in variable.supplier:
continue
nf_dns = variable.path.split('.', 1)[0]
server_name = self.objectspace.space.variables[nf_dns].doc
self.suppliers.setdefault(variable.supplier, []).append({'option': variable,
'dns': server_name,
'path_prefix': nf_dns,
'server_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names'],
'zone_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'],
'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'])
})
if not hasattr(variable, 'information'):
variable.information = self.objectspace.information(variable.xmlfiles)
variable.information.supplier = variable.supplier
def convert_providers(self):
self.providers = {}
for variable in self.get_variables():
if not hasattr(variable, 'provider'):
if not hasattr(variable, 'supplier') and not hasattr(variable, 'provider'):
continue
nf_dns = variable.path.split('.', 1)[0]
server_name = self.objectspace.space.variables[nf_dns].doc
provider_name = variable.provider
if self.objectspace.rougailconfig['risotto_globals'][server_name]['global:module_name'] == 'host':
server_names = [server_name]
else:
# supplier
if hasattr(variable, 'supplier') and ':' not in variable.supplier:
server_names = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names']
if provider_name != 'Host' and not provider_name.startswith('Host:') and not provider_name.startswith('global:'):
p_data = {'option': variable,
zones = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name']
s_data = {'option': variable,
'dns': server_name,
'path_prefix': nf_dns,
'server_names': server_names,
'zone_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'],
'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name']),
'zones': zones,
'providers_zone': {},
}
else:
p_data = None
if ':' in provider_name:
key_name, key_type = provider_name.rsplit(':', 1)
is_provider = False
else:
key_name = key_type = provider_name
is_provider = True
if provider_name != 'Host':
self.providers.setdefault(provider_name, []).append(p_data)
if key_name != 'global' and key_name not in self.suppliers:
#warn(f'cannot find supplier "{key_name}" for "{server_name}"')
continue
# create a fill for this variable
fill = self.objectspace.fill(variable.xmlfiles)
new_target = self.objectspace.target(variable.xmlfiles)
new_target.name = variable
fill.target = [new_target]
if key_name == 'global':
fill.name = 'calc_providers_global'
elif self.objectspace.paths.is_dynamic(variable):
if self.objectspace.paths.is_follower(variable):
fill.name = 'calc_providers_dynamic_follower'
if variable.supplier != 'Host' and not variable.supplier.startswith('Host:') and not variable.supplier.startswith('global:'):
if 'global:provider_zone' in self.objectspace.rougailconfig['risotto_globals'][server_name]:
s_data['provider_zone'] = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:provider_zone']
self.suppliers.setdefault(variable.supplier, []).append(s_data)
if not hasattr(variable, 'information'):
variable.information = self.objectspace.information(variable.xmlfiles)
variable.information.supplier = variable.supplier
# provider
if hasattr(variable, 'provider'):
provider_name = variable.provider
p_data = {'option': variable,
'dns': server_name,
'provider_name': provider_name,
'path_prefix': nf_dns,
'suppliers_zone': {},
}
if variable.provider != 'Host' and not variable.provider.startswith('Host:') and not variable.provider.startswith('global:'):
if 'global:provider_zone' in self.objectspace.rougailconfig['risotto_globals'][server_name]:
p_data['provider_zone'] = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:provider_zone']
if self.objectspace.rougailconfig['risotto_globals'][server_name]['global:module_name'] == 'host':
server_names = [server_name]
else:
fill.name = 'calc_providers_dynamic'
elif self.objectspace.paths.is_follower(variable):
fill.name = 'calc_providers_follower'
else:
fill.name = 'calc_providers'
fill.namespace = variable.namespace
fill.index = 0
# first parameter: the provider name (something link Host:incoming_ports)
param = self.objectspace.param(variable.xmlfiles)
param.name = 'provider'
param.text = provider_name
fill.param = [param]
# second parameter: current variable is a multi variable?
param = self.objectspace.param(variable.xmlfiles)
param.name = 'multi'
param.text = variable.multi
param.type = 'boolean'
fill.param.append(param)
if self.objectspace.paths.is_follower(variable):
server_names = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names']
p_data['server_names'] = server_names
if self.objectspace.rougailconfig['risotto_globals'][server_name]['global:module_name'] != 'host':
p_data['zones'] = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name']
if ':' in provider_name:
p_data['is_main_provider'] = False
provider = provider_name.rsplit(':', 1)[0]
else:
p_data['is_main_provider'] = True
provider = variable.provider
self.providers.setdefault(provider, []).append(p_data)
def dispatch_provider_supplier_to_zones(self):
"""calculate zone where provider and supplier communicate
"""
self.providers_zone = {}
for provider_name, p_datas in self.providers.items():
for p_data in p_datas:
if provider_name in ['global', 'Host'] or provider_name not in self.suppliers:
continue
if not 'provider_zone' in p_data:
provider_zone = None
else:
provider_zone = p_data['provider_zone']
for s_data in self.suppliers[provider_name]:
if not provider_zone:
if provider_name.endswith('Client'):
p_server = provider_name[0:-6]
if p_server not in self.providers:
continue
for p_data_t in self.providers[p_server]:
if p_data_t['dns'] == s_data['dns'] and 'provider_zone' in p_data_t:
zone = p_data_t['provider_zone']
break
else:
continue
else:
continue
else:
zone = provider_zone
if zone not in s_data['zones']:
continue
s_data['providers_zone'][provider_name] = zone
p_data['suppliers_zone'].setdefault(provider_name, {})[s_data['dns']] = zone
self.providers_zone.setdefault(zone, set()).add(provider_name)
def dispatch_provider_to_zones(self):
""" add information with provider zone domain name
"""
self.providers_zone = {}
for provider_name, p_datas in self.providers.items():
for p_data in p_datas:
if provider_name in ['global', 'Host'] or provider_name not in self.suppliers:
continue
if not 'provider_zone' in p_data:
continue
provider_zone = p_data['provider_zone']
family = self.objectspace.paths.get_variable(f"providers",
namespace=self.objectspace.rougailconfig['variable_namespace'],
force_path_prefix=p_data['path_prefix'],
)
if not hasattr(family, 'information'):
family.information = self.objectspace.information(family.xmlfiles)
name_in_zone = p_data['server_names'][p_data['zones'].index(provider_zone)]
setattr(family.information, provider_name, name_in_zone)
setattr(family.information, f'{provider_name}:zone', provider_zone)
def convert_providers(self):
for provider_name, providers_data in self.providers.items():
for provider_data in providers_data:
if provider_name != 'global' and provider_name not in self.suppliers:
continue
# create a fill for this variable
variable = provider_data['option']
fill = self.objectspace.fill(variable.xmlfiles)
new_target = self.objectspace.target(variable.xmlfiles)
new_target.name = variable
fill.target = [new_target]
if provider_name == 'global':
fill.name = 'calc_providers_global'
elif self.objectspace.paths.is_dynamic(variable):
if self.objectspace.paths.is_follower(variable):
fill.name = 'calc_providers_dynamic_follower'
else:
fill.name = 'calc_providers_dynamic'
elif self.objectspace.paths.is_follower(variable):
fill.name = 'calc_providers_follower'
else:
fill.name = 'calc_providers'
fill.namespace = variable.namespace
fill.index = 0
# first parameter: the provider name (something link Host:incoming_ports)
param = self.objectspace.param(variable.xmlfiles)
param.name = 'leader'
param.text = self.objectspace.paths.get_leader(variable)
param.propertyerror = False
param.type = 'variable'
param.name = 'provider'
param.text = provider_data['provider_name']
fill.param = [param]
# second parameter: current variable is a multi variable?
param = self.objectspace.param(variable.xmlfiles)
param.name = 'multi'
param.text = variable.multi
param.type = 'boolean'
fill.param.append(param)
try:
leader_provider = self.objectspace.paths.get_leader(variable).provider
except:
leader_provider = None
#
param = self.objectspace.param(variable.xmlfiles)
param.name = 'index'
param.type = 'index'
param.name = 'unique'
param.text = variable.unique != "False"
param.type = 'boolean'
fill.param.append(param)
if self.objectspace.paths.is_dynamic(variable):
# if dynamic: current suffix
# and add current DNS name, this is useful to known if supplier is link to this provider
param = self.objectspace.param(variable.xmlfiles)
param.name = 'suffix'
param.type = 'suffix'
fill.param.append(param)
if key_name != 'global':
param = self.objectspace.param(variable.xmlfiles)
param.name = 'dns'
param.text = server_names
fill.param.append(param)
if key_name == 'global':
param = self.objectspace.param(variable.xmlfiles)
param.name = 'value'
if provider_name in self.objectspace.rougailconfig['risotto_globals'][server_name]:
value = self.objectspace.rougailconfig['risotto_globals'][server_name][provider_name]
param.text = value
if isinstance(value, bool):
param.type = 'boolean'
else:
param.text = provider_name
param.type = 'information'
fill.param.append(param)
else:
# parse all supplier link to current provider
for idx, data in enumerate(self.suppliers[key_name]):
if p_data:
common_zones = data['zones'] & p_data['zones']
if not common_zones:
continue
for zidx, zone in enumerate(data['zone_names']):
if zone in common_zones:
break
dns = data['server_names'][zidx]
else:
dns = data['dns']
option = data['option']
# if not provider, get the true option that we want has value
if not is_provider:
path_prefix = data['path_prefix']
try:
supplier_option = self.objectspace.paths.get_supplier(f'supplier:{provider_name}', path_prefix)
except KeyError:
#warn(f'cannot find supplier "{provider_name}" for "{dns}"')
continue
# first of all, get the supplier name
#
if self.objectspace.paths.is_follower(variable):
param = self.objectspace.param(variable.xmlfiles)
param.name = f'dns_{idx}'
param.text = option
param.name = 'leader'
param.text = self.objectspace.paths.get_leader(variable)
param.propertyerror = False
param.type = 'variable'
fill.param.append(param)
if not is_provider and \
self.objectspace.paths.is_follower(variable):
try:
leader_provider = self.objectspace.paths.get_leader(variable).provider
except:
leader_provider = None
#
param = self.objectspace.param(variable.xmlfiles)
param.name = 'index'
param.type = 'index'
fill.param.append(param)
if self.objectspace.paths.is_dynamic(variable):
# if dynamic: current suffix
# and add current DNS name, this is useful to known if supplier is link to this provider
param = self.objectspace.param(variable.xmlfiles)
param.name = 'suffix'
param.type = 'suffix'
fill.param.append(param)
if provider_name != 'global':
param = self.objectspace.param(variable.xmlfiles)
param.name = 'dns'
param.text = provider_data['server_names']
fill.param.append(param)
if provider_name == 'global':
param = self.objectspace.param(variable.xmlfiles)
param.name = 'value'
if provider_data['provider_name'] in self.objectspace.rougailconfig['risotto_globals'][provider_data['dns']]:
value = self.objectspace.rougailconfig['risotto_globals'][provider_data['dns']][provider_data['provider_name']]
param.text = value
if isinstance(value, bool):
param.type = 'boolean'
else:
param.text = provider_data['provider_name']
param.type = 'information'
fill.param.append(param)
else:
# parse all supplier link to current provider
for idx, data in enumerate(self.suppliers[provider_name]):
if 'zones' in provider_data:
if provider_name not in data['providers_zone']:
continue
zone = data['providers_zone'][provider_name]
zidx = data['zones'].index(zone)
dns = data['server_names'][zidx]
else:
dns = data['dns']
option = data['option']
# if not provider, get the true option that we want have value
if not provider_data['is_main_provider']:
path_prefix = data['path_prefix']
try:
supplier_option = self.objectspace.paths.get_supplier(f'supplier:{provider_data["provider_name"]}', path_prefix)
except KeyError:
#warn(f'cannot find supplier "{provider_name}" for "{dns}"')
continue
# first of all, get the supplier name
param = self.objectspace.param(variable.xmlfiles)
param.name = f'leader_{idx}'
if fill.name == 'calc_providers_follower':
param.name = f'dns_{idx}'
param.text = option
param.propertyerror = False
param.type = 'variable'
fill.param.append(param)
if not provider_data['is_main_provider'] and \
self.objectspace.paths.is_follower(variable):
param = self.objectspace.param(variable.xmlfiles)
param.name = f'leader_{idx}'
if fill.name == 'calc_providers_follower':
param.text = dns
else:
if self.objectspace.paths.is_follower(supplier_option):
param.text = self.objectspace.paths.get_leader(supplier_option)
else:
param.text = self.objectspace.paths.get_supplier(f'supplier:{leader_provider}', path_prefix)
param.propertyerror = False
param.type = 'variable'
fill.param.append(param)
# get the current DNS name for dynamic variable
if self.objectspace.paths.is_dynamic(variable):
param = self.objectspace.param(variable.xmlfiles)
param.name = f'dynamic_{idx}'
param.text = dns
fill.param.append(param)
# get the current value!
param = self.objectspace.param(variable.xmlfiles)
param.name = f'value_{idx}'
if provider_data['is_main_provider']:
param.text = dns
else:
if self.objectspace.paths.is_follower(supplier_option):
param.text = self.objectspace.paths.get_leader(supplier_option)
else:
param.text = self.objectspace.paths.get_supplier(f'supplier:{leader_provider}', path_prefix)
param.text = supplier_option
param.propertyerror = False
param.type = 'variable'
fill.param.append(param)
# get the current DNS name for dynamic variable
if self.objectspace.paths.is_dynamic(variable):
param = self.objectspace.param(variable.xmlfiles)
param.name = f'dynamic_{idx}'
param.text = dns
fill.param.append(param)
# get the current value!
param = self.objectspace.param(variable.xmlfiles)
param.name = f'value_{idx}'
if is_provider:
param.text = dns
else:
param.text = supplier_option
param.propertyerror = False
param.type = 'variable'
fill.param.append(param)
if not hasattr(self.objectspace.space.variables[nf_dns], 'constraints'):
self.objectspace.space.variables[nf_dns].constraints = self.objectspace.constraints(None)
if not hasattr(self.objectspace.space.variables[nf_dns].constraints, 'fill'):
self.objectspace.space.variables[nf_dns].constraints.fill = []
self.objectspace.space.variables[nf_dns].constraints.fill.append(fill)
if not hasattr(self.objectspace.space.variables[provider_data['path_prefix']], 'constraints'):
self.objectspace.space.variables[provider_data['path_prefix']].constraints = self.objectspace.constraints(None)
if not hasattr(self.objectspace.space.variables[provider_data['path_prefix']].constraints, 'fill'):
self.objectspace.space.variables[provider_data['path_prefix']].constraints.fill = []
self.objectspace.space.variables[provider_data['path_prefix']].constraints.fill.append(fill)
def convert_suppliers(self):
for supplier, data in self.suppliers.items():
if supplier == 'Host':
for supplier_name, s_datas in self.suppliers.items():
if supplier_name == 'Host':
continue
for s_dico in data:
if supplier not in self.providers:
for s_data in s_datas:
if supplier_name not in self.providers:
continue
for p_dico in self.providers[supplier]:
common_zones = s_dico['zones'] & p_dico['zones']
if not common_zones:
for p_data in self.providers[supplier_name]:
if s_data['dns'] == p_data['dns']:
# supplier and provider are in same machine
continue
for idx, zone in enumerate(p_dico['zone_names']):
if zone in common_zones:
break
dns = p_dico['server_names'][idx]
s_dico['option'].value = dns
if supplier_name not in p_data['suppliers_zone'] or s_data['dns'] not in p_data['suppliers_zone'][supplier_name]:
continue
# get the DNS name in supplier zone
zone = p_data['suppliers_zone'][supplier_name][s_data['dns']]
if zone not in p_data['zones']:
continue
zidx = p_data['zones'].index(zone)
dns = p_data['server_names'][zidx]
new_value = self.objectspace.value(None)
new_value.name = dns
s_dico['option'].value = [new_value]
s_data['option'].value = [new_value]
break

64
src/risotto/utils.py
View file

@ -4,6 +4,7 @@ from typing import List
from ipaddress import ip_address
from toml import load as toml_load
from json import load, dump
from json.decoder import JSONDecodeError
from pprint import pprint
@ -15,6 +16,10 @@ HERE = environ['PWD']
IP_DIR = join(HERE, 'ip')
# custom filters from dataset
custom_filters = {}
config_file = environ.get('CONFIG_FILE', 'risotto.conf')
if isfile(config_file):
with open(config_file, 'r') as fh:
@ -35,10 +40,10 @@ def multi_function(function):
return function
async def value_pprint(dico, config):
def value_pprint(dico, config):
pprint_dict = {}
for path, value in dico.items():
if await config.option(path).option.type() == 'password' and value:
if config.option(path).type() == 'password' and value:
value = 'X' * len(value)
pprint_dict[path] = value
pprint(pprint_dict)
@ -49,28 +54,45 @@ def load_zones(zones, hosts):
makedirs(IP_DIR)
json_file = join(IP_DIR, 'zones.json')
if isfile(json_file):
with open(json_file, 'r') as fh:
zones_ip = load(fh)
try:
with open(json_file, 'r') as fh:
ori_zones_ip = load(fh)
except JSONDecodeError:
ori_zones_ip = {}
else:
zones_ip = {}
for host_name, hosts in hosts.items():
for server_name, server in hosts['servers'].items():
server_zones = server['informations']['zones_name']
ori_zones_ip = {}
new_zones_ip = {}
# cache, machine should not change IP
for host_name, dhosts in hosts.items():
for server_name, server in dhosts['servers'].items():
server_zones = server['zones_name']
for idx, zone_name in enumerate(server_zones):
zone = zones[zone_name]
zone.setdefault('hosts', {})
# FIXME make a cache, machine should not change IP
if zone_name not in zones_ip:
zones_ip[zone_name] = {}
if server_name in zones_ip[zone_name]:
server_index = zones_ip[zone_name][server_name]
elif not zones_ip[zone_name]:
server_index = 0
if zone_name not in new_zones_ip:
new_zones_ip[zone_name] = {}
if zone_name in ori_zones_ip and server_name in ori_zones_ip[zone_name]:
server_index = ori_zones_ip[zone_name][server_name]
if server_index >= zone['length']:
server_index = None
elif server_index in new_zones_ip[zone_name].values():
server_index = None
else:
# it's the last ip + 1
server_index = zones_ip[zone_name][list(zones_ip[zone_name].keys())[-1]] + 1
ip = str(ip_address(zone['start_ip']) + server_index)
zone['hosts'][server_name] = ip
zones_ip[zone_name][server_name] = server_index
server_index = None
new_zones_ip[zone_name][server_name] = server_index
for zone_name, servers in new_zones_ip.items():
for server_name, server_idx in servers.items():
if server_idx is not None:
continue
for new_idx in range(zones[zone_name]['length']):
if new_idx not in new_zones_ip[zone_name].values():
new_zones_ip[zone_name][server_name] = new_idx
break
else:
raise Exception(f'cannot find free IP in zone "{zone_name}" for "{server_name}"')
for zone_name, servers in new_zones_ip.items():
start_ip = ip_address(zones[zone_name]['start_ip'])
for server_name, server_index in servers.items():
zones[zone_name]['hosts'][server_name] = str(start_ip + server_index)
with open(json_file, 'w') as fh:
dump(zones_ip, fh)
dump(new_zones_ip, fh)