forked from stove/risotto
split code
This commit is contained in:
Emmanuel Garette
parent
23adccacb3
commit
382a5322a6
5 changed files with 834 additions and 614 deletions
585
bootstrap.py
585
bootstrap.py
|
|
@ -1,541 +1,43 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from asyncio import run
|
||||
from os import listdir, link, makedirs, environ
|
||||
from os.path import isdir, isfile, join
|
||||
from shutil import rmtree, copy2, copytree
|
||||
from json import load as json_load
|
||||
from yaml import load, SafeLoader
|
||||
from toml import load as toml_load
|
||||
from pprint import pprint
|
||||
from typing import Any
|
||||
from warnings import warn_explicit
|
||||
from os import listdir, link, makedirs
|
||||
from os.path import isdir, join
|
||||
from shutil import rmtree
|
||||
from copy import copy
|
||||
|
||||
from tiramisu import Config
|
||||
from tiramisu.error import ValueWarning
|
||||
from rougail import RougailConfig, RougailConvert, RougailSystemdTemplate
|
||||
from rougail.utils import normalize_family
|
||||
from rougail import RougailSystemdTemplate
|
||||
|
||||
from risotto.utils import MULTI_FUNCTIONS, CONFIGS, DOMAINS
|
||||
from risotto.utils import CONFIGS, RISOTTO_CONFIG, SERVERS, value_pprint
|
||||
from risotto.image import load
|
||||
|
||||
|
||||
with open(environ.get('CONFIG_FILE', 'risotto.conf'), 'r') as fh:
|
||||
config = toml_load(fh)
|
||||
|
||||
|
||||
DATASET_DIRECTORY = config['directories']['dataset']
|
||||
INSTALL_DIR = config['directories']['dest']
|
||||
FUNCTIONS = 'funcs.py'
|
||||
INSTALL_DIR = RISOTTO_CONFIG['directories']['dest']
|
||||
CONFIG_DEST_DIR = 'configurations'
|
||||
CONFIG_DIFF_DIR = 'diff'
|
||||
SRV_DEST_DIR = 'srv'
|
||||
|
||||
|
||||
with open('servers.json', 'r') as server_fh:
|
||||
jsonfile = json_load(server_fh)
|
||||
SERVERS = jsonfile['servers']
|
||||
MODULES = jsonfile['modules']
|
||||
|
||||
|
||||
async def set_linked_multi_variables(value: str,
|
||||
linked_server: str=None,
|
||||
variable_index: int=None,
|
||||
**kwargs: dict,
|
||||
) -> None:
|
||||
if value is not None and linked_server is not None and 'linked_value_0' not in kwargs:
|
||||
kwargs['linked_value_0'] = value
|
||||
elif not linked_server:
|
||||
linked_server = value
|
||||
if not linked_server:
|
||||
return
|
||||
if linked_server not in CONFIGS:
|
||||
warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
3,
|
||||
)
|
||||
return
|
||||
config = CONFIGS[linked_server][0]
|
||||
dico = {}
|
||||
variables = {}
|
||||
for key, value in kwargs.items():
|
||||
if value is None:
|
||||
return
|
||||
if key.startswith('linked_provider_'):
|
||||
index = int(key[16])
|
||||
path = await config.information.get('provider:' + value, None)
|
||||
if not path:
|
||||
return
|
||||
if index not in variables:
|
||||
variables[index] = {'path': None, 'value': None, 'variable_index': False}
|
||||
variables[index]['path'] = path
|
||||
elif key.startswith('linked_value_'):
|
||||
index = int(key[13])
|
||||
if index not in variables:
|
||||
variables[index] = {'path': None, 'value': None, 'variable_index': False}
|
||||
variables[index]['value'] = value
|
||||
elif key.startswith('variable_index_'):
|
||||
index = int(key[15])
|
||||
if index not in variables:
|
||||
variables[index] = {'path': None, 'value': None, 'variable_index': False}
|
||||
variables[index]['variable_index'] = True
|
||||
else:
|
||||
raise AttributeError(f'unknown parameter {key}')
|
||||
await config.property.read_write()
|
||||
# print('=====================================')
|
||||
# pprint(variables)
|
||||
if not isinstance(variables[0]['value'], list):
|
||||
variables[0]['value'] = [variables[0]['value']]
|
||||
dynamic = None
|
||||
try:
|
||||
if variables[0]['value']:
|
||||
for first_idx, first_value in enumerate(variables[0]['value']):
|
||||
slave_idxes = []
|
||||
dynamic = normalize_family(first_value)
|
||||
for index in sorted(list(variables)):
|
||||
path = variables[index]['path']
|
||||
if '{suffix}' in path:
|
||||
path = path.replace('{suffix}', dynamic)
|
||||
elif first_idx != 0:
|
||||
continue
|
||||
value = variables[index]['value']
|
||||
option = config.forcepermissive.option(path)
|
||||
if not await option.option.isfollower():
|
||||
#print('===>', path, value, await option.option.ismulti(), await option.option.issubmulti())
|
||||
multi = await option.option.ismulti()
|
||||
if multi:
|
||||
isleader = await option.option.isleader()
|
||||
if not isinstance(value, list):
|
||||
value = [value]
|
||||
# elif isleader:
|
||||
# raise Exception('leader must not be a multi from now ...')
|
||||
values = await option.value.get()
|
||||
for val in value:
|
||||
if val not in values:
|
||||
if isleader:
|
||||
slave_idxes.append(len(values))
|
||||
values.append(val)
|
||||
elif isleader:
|
||||
slave_idxes.append(values.index(val))
|
||||
await option.value.set(values)
|
||||
else:
|
||||
await option.value.set(value)
|
||||
else:
|
||||
#print('===<', path, value, await option.option.ismulti(), await option.option.issubmulti())
|
||||
if not slave_idxes:
|
||||
raise Exception('please declare the leader variable before the follower')
|
||||
if variables[index]['variable_index']:
|
||||
value = value[variable_index]
|
||||
if not isinstance(value, list):
|
||||
value = [value] * len(slave_idxes)
|
||||
# if isinstance(value, list) and not await option.option.issubmulti():
|
||||
for idx, val in enumerate(value):
|
||||
option = config.forcepermissive.option(path, slave_idxes[idx])
|
||||
await option.value.set(val)
|
||||
except Exception as err:
|
||||
await config.property.read_only()
|
||||
raise err from err
|
||||
await config.property.read_only()
|
||||
return get_ip_from_domain(linked_server)
|
||||
|
||||
|
||||
async def set_linked(linked_server: str,
|
||||
linked_provider: str,
|
||||
linked_value: str,
|
||||
linked_returns: str=None,
|
||||
dynamic: str=None,
|
||||
):
|
||||
if None in (linked_server, linked_provider, linked_value):
|
||||
return
|
||||
if linked_server not in CONFIGS:
|
||||
warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
0,
|
||||
)
|
||||
return
|
||||
config = CONFIGS[linked_server][0]
|
||||
path = await config.information.get('provider:' + linked_provider, None)
|
||||
if not path:
|
||||
warn_explicit(ValueWarning(f'cannot find provider "{linked_provider}" in linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
0,
|
||||
)
|
||||
return
|
||||
await config.property.read_write()
|
||||
try:
|
||||
option = config.forcepermissive.option(path)
|
||||
if await option.option.ismulti():
|
||||
values = await option.value.get()
|
||||
if linked_value not in values:
|
||||
values.append(linked_value)
|
||||
await option.value.set(values)
|
||||
else:
|
||||
await option.value.set(linked_value)
|
||||
except Exception as err:
|
||||
await config.property.read_only()
|
||||
raise err from err
|
||||
await config.property.read_only()
|
||||
if linked_returns is not None:
|
||||
linked_variable = await config.information.get('provider:' + linked_returns, None)
|
||||
if not linked_variable:
|
||||
warn_explicit(ValueWarning(f'cannot find linked variable "{linked_returns}" in linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
0,
|
||||
)
|
||||
return
|
||||
else:
|
||||
linked_variable = None
|
||||
if linked_variable is not None:
|
||||
if dynamic:
|
||||
linked_variable = linked_variable.replace('{suffix}', normalize_family(dynamic))
|
||||
elif '{suffix}' in linked_variable:
|
||||
idx = CONFIGS[linked_server][3]
|
||||
linked_variable = linked_variable.replace('{suffix}', str(idx))
|
||||
ret = await config.forcepermissive.option(linked_variable).value.get()
|
||||
else:
|
||||
ret = normalize_family(linked_value)
|
||||
return ret
|
||||
|
||||
|
||||
async def get_linked_configuration(linked_server: str,
|
||||
linked_provider: str,
|
||||
dynamic: str=None,
|
||||
):
|
||||
if linked_server not in CONFIGS:
|
||||
warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
1,
|
||||
)
|
||||
return
|
||||
config = CONFIGS[linked_server][0]
|
||||
path = await config.information.get('provider:' + linked_provider, None)
|
||||
if not path:
|
||||
warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
1,
|
||||
)
|
||||
return
|
||||
if dynamic:
|
||||
path = path.replace('{suffix}', normalize_family(dynamic))
|
||||
try:
|
||||
return await config.forcepermissive.option(path).value.get()
|
||||
except AttributeError as err:
|
||||
warn_explicit(ValueWarning(f'cannot find get value of "{path}" in linked server "{linked_server}": {err}'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
1,
|
||||
)
|
||||
|
||||
|
||||
class Empty:
|
||||
pass
|
||||
empty = Empty()
|
||||
|
||||
|
||||
async def set_linked_configuration(_linked_value: Any,
|
||||
linked_server: str,
|
||||
linked_provider: str,
|
||||
linked_value: Any=empty,
|
||||
dynamic: str=None,
|
||||
leader_provider: str=None,
|
||||
leader_value: Any=None,
|
||||
leader_index: int=None,
|
||||
):
|
||||
if linked_value is not empty:
|
||||
_linked_value = linked_value
|
||||
linked_value = _linked_value
|
||||
if linked_server is None:
|
||||
return
|
||||
if linked_value is None or linked_server not in CONFIGS:
|
||||
warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
2,
|
||||
)
|
||||
return
|
||||
config = CONFIGS[linked_server][0]
|
||||
path = await config.information.get('provider:' + linked_provider, None)
|
||||
if not path:
|
||||
warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
2,
|
||||
)
|
||||
return
|
||||
if dynamic:
|
||||
path = path.replace('{suffix}', normalize_family(dynamic))
|
||||
await config.property.read_write()
|
||||
try:
|
||||
if leader_provider is not None:
|
||||
leader_path = await config.information.get('provider:' + leader_provider, None)
|
||||
if not leader_path:
|
||||
await config.property.read_only()
|
||||
warn_explicit(ValueWarning(f'cannot find leader variable with leader_provider "{leader_provider}" in linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
2,
|
||||
)
|
||||
return
|
||||
if dynamic:
|
||||
leader_path = leader_path.replace('{suffix}', normalize_family(dynamic))
|
||||
values = await config.forcepermissive.option(leader_path).value.get()
|
||||
if not isinstance(leader_value, list):
|
||||
leader_value = [leader_value]
|
||||
for lv in leader_value:
|
||||
if lv in values:
|
||||
slave_idx = values.index(lv)
|
||||
slave_option = config.forcepermissive.option(path, slave_idx)
|
||||
if await slave_option.option.issubmulti():
|
||||
slave_values = await slave_option.value.get()
|
||||
if linked_value not in slave_values:
|
||||
slave_values.append(linked_value)
|
||||
await slave_option.value.set(slave_values)
|
||||
else:
|
||||
await slave_option.value.set(linked_value)
|
||||
else:
|
||||
option = config.forcepermissive.option(path, leader_index)
|
||||
if leader_index is None and await option.option.ismulti() and not isinstance(linked_value, list):
|
||||
values = await option.value.get()
|
||||
if linked_value not in values:
|
||||
values.append(linked_value)
|
||||
await option.value.set(values)
|
||||
else:
|
||||
await option.value.set(linked_value)
|
||||
except AttributeError as err:
|
||||
#raise ValueError(str(err)) from err
|
||||
pass
|
||||
except Exception as err:
|
||||
await config.property.read_only()
|
||||
raise err from err
|
||||
await config.property.read_only()
|
||||
|
||||
|
||||
def tiramisu_display_name(kls,
|
||||
dyn_name: 'Base'=None,
|
||||
suffix: str=None,
|
||||
) -> str:
|
||||
# FIXME
|
||||
if dyn_name is not None:
|
||||
name = kls.impl_getpath() + suffix
|
||||
name = kls.impl_getpath() + str(suffix)
|
||||
else:
|
||||
name = kls.impl_getpath()
|
||||
return name
|
||||
|
||||
|
||||
def load_applications():
|
||||
applications = {}
|
||||
distrib_dir = join(DATASET_DIRECTORY, 'applicationservice')
|
||||
for release in listdir(distrib_dir):
|
||||
release_dir = join(distrib_dir, release)
|
||||
if not isdir(release_dir):
|
||||
continue
|
||||
for applicationservice in listdir(release_dir):
|
||||
applicationservice_dir = join(release_dir, applicationservice)
|
||||
if not isdir(applicationservice_dir):
|
||||
continue
|
||||
if applicationservice in applications:
|
||||
raise Exception(f'multi applicationservice: {applicationservice} ({applicationservice_dir} <=> {applications[applicationservice]})')
|
||||
applications[applicationservice] = applicationservice_dir
|
||||
return applications
|
||||
|
||||
|
||||
class ModuleCfg():
|
||||
def __init__(self):
|
||||
self.dictionaries_dir = []
|
||||
self.modules = []
|
||||
self.functions_file = [FUNCTIONS]
|
||||
self.templates_dir = []
|
||||
self.extra_dictionaries = {}
|
||||
self.servers = []
|
||||
|
||||
|
||||
def build_module(module_name, datas, module_infos):
|
||||
install_dir = join(INSTALL_DIR, module_name)
|
||||
makedirs(install_dir)
|
||||
applications = load_applications()
|
||||
cfg = ModuleCfg()
|
||||
module_infos[module_name] = cfg
|
||||
def calc_depends(appname, added):
|
||||
if appname in added:
|
||||
return
|
||||
if appname not in applications:
|
||||
raise Exception(f'cannot find application dependency "{appname}" in application "{module_name}"')
|
||||
as_dir = applications[appname]
|
||||
cfg.modules.append(appname)
|
||||
dictionaries_dir = join(as_dir, 'dictionaries')
|
||||
if isdir(dictionaries_dir):
|
||||
cfg.dictionaries_dir.append(dictionaries_dir)
|
||||
funcs_dir = join(as_dir, 'funcs')
|
||||
if isdir(funcs_dir):
|
||||
for f in listdir(funcs_dir):
|
||||
if f.startswith('__'):
|
||||
continue
|
||||
cfg.functions_file.append(join(funcs_dir, f))
|
||||
templates_dir = join(as_dir, 'templates')
|
||||
if isdir(templates_dir):
|
||||
cfg.templates_dir.append(templates_dir)
|
||||
extras_dir = join(as_dir, 'extras')
|
||||
if isdir(extras_dir):
|
||||
for extra in listdir(extras_dir):
|
||||
extra_dir = join(extras_dir, extra)
|
||||
if isdir(extra_dir):
|
||||
cfg.extra_dictionaries.setdefault(extra, []).append(extra_dir)
|
||||
for type in ['image', 'install']:
|
||||
manual_dir = join(as_dir, 'manual', type)
|
||||
if isdir(manual_dir):
|
||||
for filename in listdir(manual_dir):
|
||||
src_file = join(manual_dir, filename)
|
||||
if type == 'image':
|
||||
dst_file = join(install_dir, 'manual', filename)
|
||||
verify = False
|
||||
else:
|
||||
dst_file= join(INSTALL_DIR, filename)
|
||||
verify = True
|
||||
if isdir(src_file):
|
||||
if not isdir(dst_file):
|
||||
makedirs(dst_file)
|
||||
for subfilename in listdir(src_file):
|
||||
if not verify or not isfile(dst_file):
|
||||
src = join(src_file, subfilename)
|
||||
dst = join(dst_file, subfilename)
|
||||
if isfile(src):
|
||||
copy2(src, dst)
|
||||
else:
|
||||
copytree(src, dst)
|
||||
elif not verify or not isfile(dst_file):
|
||||
src = join(manual_dir, filename)
|
||||
dst = dst_file
|
||||
if isfile(src):
|
||||
copy2(src, dst)
|
||||
else:
|
||||
copytree(src, dst)
|
||||
added.append(appname)
|
||||
with open(join(as_dir, 'applicationservice.yml')) as yaml:
|
||||
app = load(yaml, Loader=SafeLoader)
|
||||
|
||||
for xml in app.get('depends', []):
|
||||
calc_depends(xml, added)
|
||||
added = []
|
||||
for applicationservice in datas['applicationservices']:
|
||||
calc_depends(applicationservice, added)
|
||||
|
||||
|
||||
def get_ip_from_domain(domain):
|
||||
if not domain:
|
||||
return
|
||||
hostname, domainname = domain.split('.', 1)
|
||||
return DOMAINS[domainname][1][DOMAINS[domainname][0].index(hostname)]
|
||||
|
||||
|
||||
async def build(server_name, datas, module_infos):
|
||||
if server_name in CONFIGS:
|
||||
raise Exception(f'server "{server_name}" is duplicate')
|
||||
cfg = RougailConfig.copy()
|
||||
module_info = module_infos[datas['module']]
|
||||
module_info.servers.append(server_name)
|
||||
if datas['module'] == 'host':
|
||||
cfg['tmpfile_dest_dir'] = datas['values']['rougail.host_install_dir'] + '/host/configurations/' + server_name
|
||||
cfg['templates_dir'] = module_info.templates_dir
|
||||
cfg['dictionaries_dir'] = module_info.dictionaries_dir
|
||||
cfg['functions_file'] = module_info.functions_file
|
||||
cfg['multi_functions'] = MULTI_FUNCTIONS
|
||||
cfg['extra_dictionaries'] = module_info.extra_dictionaries
|
||||
cfg['extra_annotators'].append('risotto.rougail')
|
||||
optiondescription = {'set_linked': set_linked,
|
||||
'set_linked_multi_variables': set_linked_multi_variables,
|
||||
'get_linked_configuration': get_linked_configuration,
|
||||
'set_linked_configuration': set_linked_configuration,
|
||||
'get_ip_from_domain': get_ip_from_domain,
|
||||
}
|
||||
cfg['internal_functions'] = list(optiondescription.keys())
|
||||
try:
|
||||
eolobj = RougailConvert(cfg)
|
||||
except Exception as err:
|
||||
print(f'Try to load {module_info.modules}')
|
||||
raise err from err
|
||||
tiram_obj = eolobj.save(None)
|
||||
# if server_name == 'revprox.in.silique.fr':
|
||||
# print(tiram_obj)
|
||||
#cfg['patches_dir'] = join(test_dir, 'patches')
|
||||
cfg['tmp_dir'] = 'tmp'
|
||||
cfg['destinations_dir'] = join(INSTALL_DIR, datas['module'], CONFIG_DEST_DIR, server_name)
|
||||
if isdir('tmp'):
|
||||
rmtree('tmp')
|
||||
makedirs('tmp')
|
||||
makedirs(cfg['destinations_dir'])
|
||||
try:
|
||||
exec(tiram_obj, None, optiondescription)
|
||||
except Exception as err:
|
||||
print(tiram_obj)
|
||||
raise Exception(f'unknown error when load tiramisu object {err}') from err
|
||||
config = await Config(optiondescription['option_0'], display_name=tiramisu_display_name)
|
||||
await config.property.read_write()
|
||||
try:
|
||||
if await config.option('machine.add_srv').value.get():
|
||||
srv = join(INSTALL_DIR, SRV_DEST_DIR, server_name)
|
||||
else:
|
||||
srv = None
|
||||
except AttributeError:
|
||||
srv = None
|
||||
await config.property.read_write()
|
||||
CONFIGS[server_name] = (config, cfg, srv, 0)
|
||||
|
||||
|
||||
async def value_pprint(dico, config):
|
||||
pprint_dict = {}
|
||||
for path, value in dico.items():
|
||||
if await config.option(path).option.type() == 'password' and value:
|
||||
value = 'X' * len(value)
|
||||
pprint_dict[path] = value
|
||||
pprint(pprint_dict)
|
||||
|
||||
|
||||
async def set_values(server_name, config, datas):
|
||||
if 'informations' in datas:
|
||||
for information, value in datas['informations'].items():
|
||||
await config.information.set(information, value)
|
||||
if 'extra_domainnames' in datas['informations']:
|
||||
for idx, extra_domainname in enumerate(datas['informations']['extra_domainnames']):
|
||||
if extra_domainname in CONFIGS:
|
||||
raise Exception(f'server "{server_name}" is duplicate')
|
||||
value = list(CONFIGS[server_name])
|
||||
value[3] = idx + 1
|
||||
CONFIGS[extra_domainname] = tuple(value)
|
||||
await config.information.set('server_name', server_name)
|
||||
await config.property.read_write()
|
||||
try:
|
||||
if 'values' in datas:
|
||||
for path, value in datas['values'].items():
|
||||
if isinstance(value, dict):
|
||||
for idx, val in value.items():
|
||||
await config.option(path, int(idx)).value.set(val)
|
||||
else:
|
||||
await config.option(path).value.set(value)
|
||||
except Exception as err:
|
||||
await value_pprint(await config.value.dict(), config)
|
||||
error_msg = f'cannot configure server "{server_name}": {err}'
|
||||
raise Exception(error_msg) from err
|
||||
await config.property.read_only()
|
||||
#await config.value.dict()
|
||||
|
||||
|
||||
async def valid_mandatories(server_name, config):
|
||||
mandatories = await config.value.mandatory()
|
||||
if mandatories:
|
||||
print()
|
||||
print(f'=== Configuration: {server_name} ===')
|
||||
await config.property.pop('mandatory')
|
||||
await value_pprint(await config.value.dict(), config)
|
||||
raise Exception(f'server "{server_name}" has mandatories variables without values "{", ".join(mandatories)}"')
|
||||
|
||||
|
||||
async def templates(server_name, config, cfg, srv, int_idx):
|
||||
async def templates(server_name,
|
||||
config,
|
||||
templates_informations,
|
||||
srv=False,
|
||||
**kwargs,
|
||||
):
|
||||
values = await config.value.dict()
|
||||
engine = RougailSystemdTemplate(config, cfg)
|
||||
engine = RougailSystemdTemplate(config, templates_informations)
|
||||
# if server_name == 'dovecot.in.silique.fr':
|
||||
# print()
|
||||
# print(f'=== Configuration: {server_name} ===')
|
||||
|
|
@ -555,30 +57,43 @@ async def main():
|
|||
if isdir(INSTALL_DIR):
|
||||
rmtree(INSTALL_DIR)
|
||||
makedirs(INSTALL_DIR)
|
||||
module_infos = {}
|
||||
for module_name, datas in MODULES.items():
|
||||
build_module(module_name, datas, module_infos)
|
||||
for server_name, datas in SERVERS.items():
|
||||
await build(server_name, datas, module_infos)
|
||||
module_infos = await load(display_name=tiramisu_display_name, clean_directories=True, copy_manual_dir=True)
|
||||
# pprint(await CONFIGS['lemonldap.in.silique.fr'][0].value.dict())
|
||||
for server_name in SERVERS:
|
||||
module_name = CONFIGS[server_name]['module_name']
|
||||
add_srv = CONFIGS[server_name]['add_srv']
|
||||
cfg = CONFIGS[server_name]['templates_informations']
|
||||
cfg['tmp_dir'] = 'tmp'
|
||||
cfg['destinations_dir'] = join(INSTALL_DIR, module_name, CONFIG_DEST_DIR, server_name)
|
||||
if isdir('tmp'):
|
||||
rmtree('tmp')
|
||||
makedirs(cfg['tmp_dir'])
|
||||
makedirs(cfg['destinations_dir'])
|
||||
if add_srv:
|
||||
srv = join(INSTALL_DIR, SRV_DEST_DIR, server_name)
|
||||
else:
|
||||
srv = None
|
||||
await templates(server_name, **CONFIGS[server_name], srv=srv)
|
||||
for server_name in SERVERS:
|
||||
config = CONFIGS[server_name]['config']
|
||||
await config.property.read_write()
|
||||
try:
|
||||
# pass
|
||||
await config.option('general.hide_secret').value.set(True)
|
||||
except AttributeError:
|
||||
# if rougail.general.hide_secret not exists
|
||||
pass
|
||||
await config.property.read_only()
|
||||
for server_name in SERVERS:
|
||||
module_name = CONFIGS[server_name]['module_name']
|
||||
destinations_dir = join(INSTALL_DIR, module_name, CONFIG_DIFF_DIR, server_name)
|
||||
makedirs(destinations_dir)
|
||||
CONFIGS[server_name]['templates_informations']['destinations_dir'] = destinations_dir
|
||||
await templates(server_name, **CONFIGS[server_name])
|
||||
for module_name, cfg in module_infos.items():
|
||||
with open(join(INSTALL_DIR, module_name, 'install_machines'), 'w') as fh:
|
||||
for server_name in cfg.servers:
|
||||
fh.write(f'./install_machine {module_name} {server_name}\n')
|
||||
for server_name, datas in SERVERS.items():
|
||||
await set_values(server_name, CONFIGS[server_name][0], datas)
|
||||
for server_name in SERVERS:
|
||||
config = CONFIGS[server_name][0]
|
||||
await config.property.pop('mandatory')
|
||||
try:
|
||||
await config.value.dict()
|
||||
except Exception as err:
|
||||
raise Exception(f'cannot display config for "{server_name}": {err}')
|
||||
await config.property.add('mandatory')
|
||||
for server_name in SERVERS:
|
||||
await valid_mandatories(server_name, CONFIGS[server_name][0])
|
||||
# print(await CONFIGS['dovecot.in.silique.fr'][0].value.dict())
|
||||
for server_name in SERVERS:
|
||||
await templates(server_name, *CONFIGS[server_name])
|
||||
|
||||
|
||||
run(main())
|
||||
|
|
|
|||
105
funcs.py
105
funcs.py
|
|
@ -1,88 +1,24 @@
|
|||
from tiramisu import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value
|
||||
from ipaddress import ip_address
|
||||
from os.path import dirname, abspath, join as _join, isdir as _isdir, isfile as _isfile
|
||||
from typing import List
|
||||
from json import load
|
||||
from secrets import token_urlsafe as _token_urlsafe
|
||||
|
||||
from rougail.utils import normalize_family
|
||||
|
||||
from risotto.utils import multi_function, CONFIGS, DOMAINS
|
||||
from risotto.utils import multi_function, DOMAINS, ZONES, load_zones, load_zones_server, load_domains, ZONES_SERVER
|
||||
from risotto.x509 import gen_cert as _x509_gen_cert, gen_ca as _x509_gen_ca, gen_pub as _x509_gen_pub, has_pub as _x509_has_pub
|
||||
# =============================================================
|
||||
# fork of risotto-setting/src/risotto_setting/config/config.py
|
||||
|
||||
with open('servers.json', 'r') as server_fh:
|
||||
ZONES_SERVER = load(server_fh)
|
||||
|
||||
|
||||
ZONES = None
|
||||
HERE = dirname(abspath(__file__))
|
||||
|
||||
|
||||
def load_zones():
|
||||
global ZONES
|
||||
if ZONES is not None:
|
||||
return
|
||||
ZONES = ZONES_SERVER['zones']
|
||||
for server_name, server in ZONES_SERVER['servers'].items():
|
||||
if 'informations' not in server:
|
||||
continue
|
||||
server_zones = server['informations']['zones_name']
|
||||
server_extra_domainnames = server['informations'].get('extra_domainnames', [])
|
||||
if len(server_zones) > 1 and len(server_zones) != len(server_extra_domainnames) + 1:
|
||||
raise Exception(f'the server "{server_name}" has more that one zone, please set correct number of extra_domainnames ({len(server_zones) - 1} instead of {len(server_extra_domainnames)})')
|
||||
|
||||
for idx, zone_name in enumerate(server_zones):
|
||||
zone_domain_name = ZONES[zone_name]['domain_name']
|
||||
if idx == 0:
|
||||
zone_server_name = server_name
|
||||
else:
|
||||
zone_server_name = server_extra_domainnames[idx - 1]
|
||||
server_domain_name = zone_server_name.split('.', 1)[1]
|
||||
if zone_domain_name and zone_domain_name != server_domain_name:
|
||||
raise Exception(f'wrong server_name "{zone_server_name}" in zone "{zone_name}" should ends with "{zone_domain_name}"')
|
||||
ZONES[zone_name].setdefault('hosts', []).append(server_name)
|
||||
|
||||
|
||||
def load_domains():
|
||||
load_zones()
|
||||
global DOMAINS
|
||||
if DOMAINS:
|
||||
return
|
||||
for zone_name, zone in ZONES_SERVER['zones'].items():
|
||||
if 'domain_name' in zone:
|
||||
hosts = []
|
||||
ips = []
|
||||
for host in ZONES[zone_name].get('hosts', []):
|
||||
hosts.append(host.split('.', 1)[0])
|
||||
ips.append(get_ip(host, [zone_name], 0))
|
||||
DOMAINS[zone['domain_name']] = (tuple(hosts), tuple(ips))
|
||||
|
||||
|
||||
def get_ip(server_name: str,
|
||||
zones_name: List[str],
|
||||
index: str,
|
||||
) -> str:
|
||||
if server_name is None:
|
||||
return
|
||||
load_zones()
|
||||
index = int(index)
|
||||
zone_name = zones_name[index]
|
||||
if zone_name not in ZONES:
|
||||
raise ValueError(f"cannot set IP in unknown zone '{zone_name}'")
|
||||
zone = ZONES[zone_name]
|
||||
if server_name not in zone['hosts']:
|
||||
raise ValueError(f"cannot set IP in unknown server '{server_name}'")
|
||||
server_index = zone['hosts'].index(server_name)
|
||||
# print(server_name, zones_name, index, str(ip_address(zone['start_ip']) + server_index))
|
||||
return str(ip_address(zone['start_ip']) + server_index)
|
||||
|
||||
|
||||
@multi_function
|
||||
def get_chain(authority_cn,
|
||||
authority_name,
|
||||
def get_chain(authority_cn: str,
|
||||
authority_name: str,
|
||||
hide: bool,
|
||||
):
|
||||
if hide:
|
||||
return "XXXXX"
|
||||
if not authority_name or authority_name is None:
|
||||
if isinstance(authority_name, list):
|
||||
return []
|
||||
|
|
@ -107,11 +43,14 @@ def get_chain(authority_cn,
|
|||
|
||||
@multi_function
|
||||
def get_certificate(cn,
|
||||
authority_name,
|
||||
authority_cn=None,
|
||||
extra_domainnames=[],
|
||||
type='server',
|
||||
authority_name: str,
|
||||
hide: bool,
|
||||
authority_cn: str=None,
|
||||
extra_domainnames: list=[],
|
||||
type: str='server',
|
||||
):
|
||||
if hide:
|
||||
return "XXXXX"
|
||||
if isinstance(cn, list) and extra_domainnames:
|
||||
raise Exception('cn cannot be a list with extra_domainnames set')
|
||||
if not cn or authority_name is None:
|
||||
|
|
@ -129,11 +68,14 @@ def get_certificate(cn,
|
|||
|
||||
|
||||
@multi_function
|
||||
def get_private_key(cn,
|
||||
authority_name=None,
|
||||
authority_cn=None,
|
||||
type='server',
|
||||
def get_private_key(cn: str,
|
||||
hide: bool,
|
||||
authority_name: str=None,
|
||||
authority_cn: str=None,
|
||||
type: str='server',
|
||||
):
|
||||
if hide:
|
||||
return "XXXXX"
|
||||
if not cn:
|
||||
if isinstance(cn, list):
|
||||
return []
|
||||
|
|
@ -157,7 +99,11 @@ def get_private_key(cn,
|
|||
)
|
||||
|
||||
|
||||
def get_public_key(cn):
|
||||
def get_public_key(cn: str,
|
||||
hide: bool,
|
||||
):
|
||||
if hide:
|
||||
return "XXXXX"
|
||||
if not cn:
|
||||
return
|
||||
return _x509_gen_pub(cn,
|
||||
|
|
@ -194,6 +140,7 @@ def get_internal_zones() -> List[str]:
|
|||
|
||||
@multi_function
|
||||
def get_zones_info(type: str) -> str:
|
||||
load_zones_server()
|
||||
ret = []
|
||||
for data in ZONES_SERVER['zones'].values():
|
||||
ret.append(data[type])
|
||||
|
|
|
|||
272
src/risotto/image.py
Normal file
272
src/risotto/image.py
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
from shutil import copy2, copytree, rmtree
|
||||
from os import listdir, makedirs
|
||||
from os.path import join, isdir, isfile
|
||||
from yaml import load as yaml_load, SafeLoader
|
||||
from json import load as json_load
|
||||
|
||||
from .utils import CONFIGS, RISOTTO_CONFIG, SERVERS, value_pprint
|
||||
from .machine import load_machine_config
|
||||
|
||||
|
||||
FUNCTIONS = 'funcs.py'
|
||||
|
||||
|
||||
class ModuleCfg():
|
||||
def __init__(self):
|
||||
self.dictionaries_dir = []
|
||||
self.modules = []
|
||||
self.functions_file = [FUNCTIONS]
|
||||
self.templates_dir = []
|
||||
self.extra_dictionaries = {}
|
||||
self.servers = []
|
||||
|
||||
|
||||
def list_applications() -> dict:
|
||||
"""
|
||||
{<applicationservice>: applicationservice/<release>/<applicationservice>
|
||||
"""
|
||||
dataset_directory = RISOTTO_CONFIG['directories']['dataset']
|
||||
applications = {}
|
||||
distrib_dir = join(dataset_directory, 'applicationservice')
|
||||
for release in listdir(distrib_dir):
|
||||
release_dir = join(distrib_dir, release)
|
||||
if not isdir(release_dir):
|
||||
continue
|
||||
for applicationservice in listdir(release_dir):
|
||||
applicationservice_dir = join(release_dir, applicationservice)
|
||||
if not isdir(applicationservice_dir):
|
||||
continue
|
||||
if applicationservice in applications:
|
||||
raise Exception(f'multi applicationservice: {applicationservice} ({applicationservice_dir} <=> {applications[applicationservice]})')
|
||||
applications[applicationservice] = applicationservice_dir
|
||||
return applications
|
||||
|
||||
|
||||
def applicationservice_copy(src_file: str,
|
||||
dst_file: str,
|
||||
copy_if_not_exists: bool,
|
||||
manual_dir: str,
|
||||
filename: str,
|
||||
) -> None:
|
||||
if isdir(src_file):
|
||||
if not isdir(dst_file):
|
||||
makedirs(dst_file)
|
||||
for subfilename in listdir(src_file):
|
||||
if not copy_if_not_exists or not isfile(dst_file):
|
||||
src = join(src_file, subfilename)
|
||||
dst = join(dst_file, subfilename)
|
||||
if isfile(src):
|
||||
copy2(src, dst)
|
||||
else:
|
||||
copytree(src, dst)
|
||||
elif not copy_if_not_exists or not isfile(dst_file):
|
||||
src = join(manual_dir, filename)
|
||||
dst = dst_file
|
||||
if isfile(src):
|
||||
copy2(src, dst)
|
||||
else:
|
||||
copytree(src, dst)
|
||||
|
||||
|
||||
def load_applicationservice_cfg(appname: str,
|
||||
as_dir: str,
|
||||
install_dir: str,
|
||||
cfg: ModuleCfg,
|
||||
copy_manual_dir: bool,
|
||||
) -> None:
|
||||
cfg.modules.append(appname)
|
||||
# dictionaries
|
||||
dictionaries_dir = join(as_dir, 'dictionaries')
|
||||
if isdir(dictionaries_dir):
|
||||
cfg.dictionaries_dir.append(dictionaries_dir)
|
||||
# funcs
|
||||
funcs_dir = join(as_dir, 'funcs')
|
||||
if isdir(funcs_dir):
|
||||
for f in listdir(funcs_dir):
|
||||
if f.startswith('__'):
|
||||
continue
|
||||
cfg.functions_file.append(join(funcs_dir, f))
|
||||
# templates
|
||||
templates_dir = join(as_dir, 'templates')
|
||||
if isdir(templates_dir):
|
||||
cfg.templates_dir.append(templates_dir)
|
||||
# extras
|
||||
extras_dir = join(as_dir, 'extras')
|
||||
if isdir(extras_dir):
|
||||
for extra in listdir(extras_dir):
|
||||
extra_dir = join(extras_dir, extra)
|
||||
if isdir(extra_dir):
|
||||
cfg.extra_dictionaries.setdefault(extra, []).append(extra_dir)
|
||||
if copy_manual_dir:
|
||||
# manual
|
||||
for type in ['image', 'install']:
|
||||
manual_dir = join(as_dir, 'manual', type)
|
||||
if not isdir(manual_dir):
|
||||
continue
|
||||
for filename in listdir(manual_dir):
|
||||
src_file = join(manual_dir, filename)
|
||||
if type == 'image':
|
||||
dst_file = join(install_dir, 'manual', filename)
|
||||
copy_if_not_exists = False
|
||||
else:
|
||||
dst_file= join(install_dir, '..', filename)
|
||||
copy_if_not_exists = True
|
||||
applicationservice_copy(src_file,
|
||||
dst_file,
|
||||
copy_if_not_exists,
|
||||
manual_dir,
|
||||
filename,
|
||||
)
|
||||
|
||||
|
||||
def load_applicationservice(appname: str,
|
||||
added: list,
|
||||
install_dir: str,
|
||||
cfg: ModuleCfg,
|
||||
applications: dict,
|
||||
copy_manual_dir: bool,
|
||||
) -> None:
|
||||
if appname not in applications:
|
||||
raise Exception(f'cannot find application dependency "{appname}" in application "{module_name}"')
|
||||
as_dir = applications[appname]
|
||||
applicationservice_file = join(as_dir, 'applicationservice.yml')
|
||||
if not isfile(applicationservice_file):
|
||||
raise Exception(f'cannot find application service file "{applicationservice_file}"')
|
||||
load_applicationservice_cfg(appname,
|
||||
as_dir,
|
||||
install_dir,
|
||||
cfg,
|
||||
copy_manual_dir,
|
||||
)
|
||||
added.append(appname)
|
||||
with open(applicationservice_file) as yaml:
|
||||
app = yaml_load(yaml, Loader=SafeLoader)
|
||||
for xml in app.get('depends', []):
|
||||
if xml in added:
|
||||
continue
|
||||
load_applicationservice(xml,
|
||||
added,
|
||||
install_dir,
|
||||
cfg,
|
||||
applications,
|
||||
copy_manual_dir,
|
||||
)
|
||||
|
||||
|
||||
def load_image_informations(install_dir: str,
|
||||
datas: dict,
|
||||
applications: dict,
|
||||
copy_manual_dir: bool,
|
||||
) -> ModuleCfg:
|
||||
cfg = ModuleCfg()
|
||||
added = []
|
||||
for applicationservice in datas['applicationservices']:
|
||||
load_applicationservice(applicationservice,
|
||||
added,
|
||||
install_dir,
|
||||
cfg,
|
||||
applications,
|
||||
copy_manual_dir,
|
||||
)
|
||||
return cfg
|
||||
|
||||
|
||||
async def load_informations(server_name, datas, config):
|
||||
await config.information.set('server_name', server_name)
|
||||
if 'informations' not in datas:
|
||||
return
|
||||
for information, value in datas['informations'].items():
|
||||
await config.information.set(information, value)
|
||||
if 'extra_domainnames' in datas['informations']:
|
||||
for idx, extra_domainname in enumerate(datas['informations']['extra_domainnames']):
|
||||
if extra_domainname in CONFIGS:
|
||||
raise Exception(f'server "{server_name}" is duplicate')
|
||||
value = list(CONFIGS[server_name])
|
||||
value[4] = idx + 1
|
||||
CONFIGS[extra_domainname] = tuple(value)
|
||||
|
||||
|
||||
async def set_values(server_name, config, datas):
|
||||
try:
|
||||
if 'values' in datas:
|
||||
for path, value in datas['values'].items():
|
||||
if isinstance(value, dict):
|
||||
for idx, val in value.items():
|
||||
await config.option(path, int(idx)).value.set(val)
|
||||
else:
|
||||
await config.option(path).value.set(value)
|
||||
except Exception as err:
|
||||
await value_pprint(await config.value.dict(), config)
|
||||
error_msg = f'cannot configure server "{server_name}": {err}'
|
||||
raise Exception(error_msg) from err
|
||||
#await config.value.dict()
|
||||
|
||||
|
||||
async def valid_mandatories(server_name, config):
|
||||
mandatories = await config.value.mandatory()
|
||||
if mandatories:
|
||||
print()
|
||||
print(f'=== Configuration: {server_name} ===')
|
||||
await config.property.pop('mandatory')
|
||||
await value_pprint(await config.value.dict(), config)
|
||||
raise Exception(f'server "{server_name}" has mandatories variables without values "{", ".join(mandatories)}"')
|
||||
|
||||
|
||||
async def load(display_name=None,
|
||||
clean_directories=False,
|
||||
copy_manual_dir=False,
|
||||
hide_secret=False,
|
||||
):
|
||||
with open('servers.json', 'r') as server_fh:
|
||||
jsonfile = json_load(server_fh)
|
||||
SERVERS.update(jsonfile['servers'])
|
||||
modules = jsonfile['modules']
|
||||
module_infos = {}
|
||||
applications = list_applications()
|
||||
# load images
|
||||
for module_name, datas in modules.items():
|
||||
install_dir = join(RISOTTO_CONFIG['directories']['dest'], module_name)
|
||||
if clean_directories:
|
||||
if isdir(install_dir):
|
||||
rmtree(install_dir)
|
||||
makedirs(install_dir)
|
||||
module_infos[module_name] = load_image_informations(install_dir,
|
||||
datas,
|
||||
applications,
|
||||
copy_manual_dir,
|
||||
)
|
||||
# load machines
|
||||
for server_name, datas in SERVERS.items():
|
||||
if server_name in CONFIGS:
|
||||
raise Exception(f'server "{server_name}" is duplicate')
|
||||
CONFIGS[server_name] = await load_machine_config(server_name,
|
||||
datas,
|
||||
module_infos[datas['module']],
|
||||
display_name=display_name,
|
||||
)
|
||||
# set servers.json values
|
||||
for server_name, datas in SERVERS.items():
|
||||
config = CONFIGS[server_name]['config']
|
||||
await load_informations(server_name, datas, config)
|
||||
await config.property.read_write()
|
||||
if hide_secret:
|
||||
try:
|
||||
await config.option('general.hide_secret').value.set(True)
|
||||
except AttributeError:
|
||||
pass
|
||||
await set_values(server_name, config, datas)
|
||||
await config.property.read_only()
|
||||
# force calculates all values (for linked values)
|
||||
for server_name in SERVERS:
|
||||
config = CONFIGS[server_name]['config']
|
||||
await config.property.pop('mandatory')
|
||||
try:
|
||||
await config.value.dict()
|
||||
except Exception as err:
|
||||
raise Exception(f'cannot display config for "{server_name}": {err}')
|
||||
await config.property.add('mandatory')
|
||||
# validate mandatories values
|
||||
for server_name in SERVERS:
|
||||
await valid_mandatories(server_name, CONFIGS[server_name]['config'])
|
||||
|
||||
return module_infos
|
||||
394
src/risotto/machine.py
Normal file
394
src/risotto/machine.py
Normal file
|
|
@ -0,0 +1,394 @@
|
|||
from os import makedirs
|
||||
from os.path import join, isdir
|
||||
from warnings import warn_explicit
|
||||
from typing import Any
|
||||
|
||||
from tiramisu import Config
|
||||
from tiramisu.error import ValueWarning
|
||||
from rougail import RougailConfig, RougailConvert
|
||||
from .utils import MULTI_FUNCTIONS, CONFIGS, DOMAINS
|
||||
from rougail.utils import normalize_family
|
||||
|
||||
|
||||
ROUGAIL_NAMESPACE = 'general'
|
||||
ROUGAIL_NAMESPACE_DESCRIPTION = 'Général'
|
||||
|
||||
|
||||
async def set_linked_multi_variables(value: str,
|
||||
linked_server: str=None,
|
||||
variable_index: int=None,
|
||||
linked_returns: str=None,
|
||||
dynamic: str=None,
|
||||
**kwargs: dict,
|
||||
) -> None:
|
||||
if value is not None and linked_server is not None and 'linked_value_0' not in kwargs:
|
||||
kwargs['linked_value_0'] = value
|
||||
elif not linked_server:
|
||||
linked_server = value
|
||||
if not linked_server:
|
||||
return
|
||||
if linked_server not in CONFIGS:
|
||||
warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
3,
|
||||
)
|
||||
return
|
||||
config = CONFIGS[linked_server]['config']
|
||||
dico = {}
|
||||
variables = {}
|
||||
for key, kvalue in kwargs.items():
|
||||
try:
|
||||
index = int(key.rsplit('_', 1)[-1])
|
||||
except ValueError:
|
||||
raise Exception(f'unknown variable {key}')
|
||||
if kvalue is None and not kwargs.get(f'allow_none_{index}', False):
|
||||
return
|
||||
if key.startswith('linked_provider_'):
|
||||
path = await config.information.get('provider:' + kvalue, None)
|
||||
if not path:
|
||||
return
|
||||
if index not in variables:
|
||||
variables[index] = {'path': None, 'value': None, 'variable_index': False}
|
||||
variables[index]['path'] = path
|
||||
elif key.startswith('linked_value_'):
|
||||
index = int(key[13])
|
||||
if index not in variables:
|
||||
variables[index] = {'path': None, 'value': None, 'variable_index': False}
|
||||
variables[index]['value'] = kvalue
|
||||
elif key.startswith('variable_index_'):
|
||||
index = int(key[15])
|
||||
if index not in variables:
|
||||
variables[index] = {'path': None, 'value': None, 'variable_index': False}
|
||||
variables[index]['variable_index'] = True
|
||||
elif key.startswith('allow_none_'):
|
||||
pass
|
||||
else:
|
||||
raise AttributeError(f'unknown parameter {key}')
|
||||
await config.property.read_write()
|
||||
if not isinstance(variables[0]['value'], list):
|
||||
variables[0]['value'] = [variables[0]['value']]
|
||||
if dynamic:
|
||||
dynamic = normalize_family(dynamic)
|
||||
_dynamic = None
|
||||
try:
|
||||
if variables[0]['value']:
|
||||
for first_idx, first_value in enumerate(variables[0]['value']):
|
||||
slave_idxes = []
|
||||
if dynamic:
|
||||
_dynamic = dynamic
|
||||
else:
|
||||
_dynamic = normalize_family(first_value)
|
||||
for index in sorted(list(variables)):
|
||||
path = variables[index]['path']
|
||||
if '{suffix}' in path:
|
||||
path = path.replace('{suffix}', _dynamic)
|
||||
elif first_idx != 0:
|
||||
continue
|
||||
vvalue = variables[index]['value']
|
||||
option = config.forcepermissive.option(path)
|
||||
if not await option.option.isfollower():
|
||||
#print('===>', path, vvalue, await option.option.ismulti(), await option.option.issubmulti())
|
||||
multi = await option.option.ismulti()
|
||||
if multi:
|
||||
isleader = await option.option.isleader()
|
||||
if not isinstance(vvalue, list):
|
||||
vvalue = [vvalue]
|
||||
# elif isleader:
|
||||
# raise Exception('leader must not be a multi from now ...')
|
||||
values = await option.value.get()
|
||||
for val in vvalue:
|
||||
if val not in values:
|
||||
if isleader:
|
||||
slave_idxes.append(len(values))
|
||||
values.append(val)
|
||||
elif isleader:
|
||||
slave_idxes.append(values.index(val))
|
||||
await option.value.set(values)
|
||||
await option.owner.set('link')
|
||||
else:
|
||||
if isinstance(vvalue, list) and len(vvalue) == 1:
|
||||
vvalue = vvalue[0]
|
||||
await option.value.set(vvalue)
|
||||
await option.owner.set('link')
|
||||
else:
|
||||
# print('===<', path, vvalue, await option.option.ismulti(), await option.option.issubmulti())
|
||||
if not slave_idxes:
|
||||
raise Exception('please declare the leader variable before the follower')
|
||||
if variables[index]['variable_index']:
|
||||
vvalue = vvalue[variable_index]
|
||||
if not isinstance(vvalue, list):
|
||||
vvalue = [vvalue] * len(slave_idxes)
|
||||
# if isinstance(vvalue, list) and not await option.option.issubmulti():
|
||||
for idx, val in enumerate(vvalue):
|
||||
option = config.forcepermissive.option(path, slave_idxes[idx])
|
||||
await option.value.set(val)
|
||||
await option.owner.set('link')
|
||||
except Exception as err:
|
||||
await config.property.read_only()
|
||||
raise err from err
|
||||
await config.property.read_only()
|
||||
if not dynamic:
|
||||
dynamic = _dynamic
|
||||
|
||||
|
||||
if linked_returns is not None:
|
||||
linked_variable = await config.information.get('provider:' + linked_returns, None)
|
||||
if not linked_variable:
|
||||
warn_explicit(ValueWarning(f'cannot find linked variable "{linked_returns}" in linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
0,
|
||||
)
|
||||
return
|
||||
if dynamic:
|
||||
linked_variable = linked_variable.replace('{suffix}', normalize_family(dynamic))
|
||||
elif '{suffix}' in linked_variable:
|
||||
idx = CONFIGS[linked_server]['interface_index']
|
||||
linked_variable = linked_variable.replace('{suffix}', str(idx))
|
||||
ret = await config.forcepermissive.option(linked_variable).value.get()
|
||||
else:
|
||||
ret = get_ip_from_domain(linked_server)
|
||||
return ret
|
||||
|
||||
|
||||
async def set_linked(linked_server: str,
|
||||
linked_provider: str,
|
||||
linked_value: str,
|
||||
linked_returns: str=None,
|
||||
dynamic: str=None,
|
||||
):
|
||||
if None in (linked_server, linked_provider, linked_value):
|
||||
return
|
||||
if linked_server not in CONFIGS:
|
||||
warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
0,
|
||||
)
|
||||
return
|
||||
config = CONFIGS[linked_server]['config']
|
||||
path = await config.information.get('provider:' + linked_provider, None)
|
||||
if not path:
|
||||
warn_explicit(ValueWarning(f'cannot find provider "{linked_provider}" in linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
0,
|
||||
)
|
||||
return
|
||||
await config.property.read_write()
|
||||
try:
|
||||
option = config.forcepermissive.option(path)
|
||||
if await option.option.ismulti():
|
||||
values = await option.value.get()
|
||||
if linked_value not in values:
|
||||
values.append(linked_value)
|
||||
await option.value.set(values)
|
||||
await option.owner.set('link')
|
||||
else:
|
||||
await option.value.set(linked_value)
|
||||
await option.owner.set('link')
|
||||
except Exception as err:
|
||||
await config.property.read_only()
|
||||
raise err from err
|
||||
await config.property.read_only()
|
||||
if linked_returns is not None:
|
||||
linked_variable = await config.information.get('provider:' + linked_returns, None)
|
||||
if not linked_variable:
|
||||
warn_explicit(ValueWarning(f'cannot find linked variable "{linked_returns}" in linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
0,
|
||||
)
|
||||
return
|
||||
else:
|
||||
linked_variable = None
|
||||
if linked_variable is not None:
|
||||
if dynamic:
|
||||
linked_variable = linked_variable.replace('{suffix}', normalize_family(dynamic))
|
||||
elif '{suffix}' in linked_variable:
|
||||
idx = CONFIGS[linked_server]['interface_index']
|
||||
linked_variable = linked_variable.replace('{suffix}', str(idx))
|
||||
ret = await config.forcepermissive.option(linked_variable).value.get()
|
||||
else:
|
||||
ret = normalize_family(linked_value)
|
||||
return ret
|
||||
|
||||
|
||||
async def get_linked_configuration(linked_server: str,
|
||||
linked_provider: str,
|
||||
dynamic: str=None,
|
||||
):
|
||||
if linked_server not in CONFIGS:
|
||||
warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
1,
|
||||
)
|
||||
return
|
||||
config = CONFIGS[linked_server]['config']
|
||||
path = await config.information.get('provider:' + linked_provider, None)
|
||||
if not path:
|
||||
warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
1,
|
||||
)
|
||||
return
|
||||
if dynamic:
|
||||
path = path.replace('{suffix}', normalize_family(dynamic))
|
||||
try:
|
||||
return await config.forcepermissive.option(path).value.get()
|
||||
except AttributeError as err:
|
||||
warn_explicit(ValueWarning(f'cannot find get value of "{path}" in linked server "{linked_server}": {err}'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
1,
|
||||
)
|
||||
|
||||
|
||||
class Empty:
|
||||
pass
|
||||
empty = Empty()
|
||||
|
||||
|
||||
async def set_linked_configuration(_linked_value: Any,
|
||||
linked_server: str,
|
||||
linked_provider: str,
|
||||
linked_value: Any=empty,
|
||||
dynamic: str=None,
|
||||
leader_provider: str=None,
|
||||
leader_value: Any=None,
|
||||
leader_index: int=None,
|
||||
):
|
||||
if linked_value is not empty:
|
||||
_linked_value = linked_value
|
||||
linked_value = _linked_value
|
||||
if linked_server is None:
|
||||
return
|
||||
if linked_value is None or linked_server not in CONFIGS:
|
||||
warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
2,
|
||||
)
|
||||
return
|
||||
config = CONFIGS[linked_server]['config']
|
||||
path = await config.information.get('provider:' + linked_provider, None)
|
||||
if not path:
|
||||
warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
2,
|
||||
)
|
||||
return
|
||||
if dynamic:
|
||||
path = path.replace('{suffix}', normalize_family(dynamic))
|
||||
await config.property.read_write()
|
||||
try:
|
||||
if leader_provider is not None:
|
||||
leader_path = await config.information.get('provider:' + leader_provider, None)
|
||||
if not leader_path:
|
||||
await config.property.read_only()
|
||||
warn_explicit(ValueWarning(f'cannot find leader variable with leader_provider "{leader_provider}" in linked server "{linked_server}"'),
|
||||
ValueWarning,
|
||||
__file__,
|
||||
2,
|
||||
)
|
||||
return
|
||||
if dynamic:
|
||||
leader_path = leader_path.replace('{suffix}', normalize_family(dynamic))
|
||||
values = await config.forcepermissive.option(leader_path).value.get()
|
||||
if not isinstance(leader_value, list):
|
||||
leader_value = [leader_value]
|
||||
for lv in leader_value:
|
||||
if lv in values:
|
||||
slave_idx = values.index(lv)
|
||||
slave_option = config.forcepermissive.option(path, slave_idx)
|
||||
if await slave_option.option.issubmulti():
|
||||
slave_values = await slave_option.value.get()
|
||||
if linked_value not in slave_values:
|
||||
slave_values.append(linked_value)
|
||||
await slave_option.value.set(slave_values)
|
||||
await slave_option.owner.set('link')
|
||||
else:
|
||||
await slave_option.value.set(linked_value)
|
||||
await slave_option.owner.set('link')
|
||||
else:
|
||||
option = config.forcepermissive.option(path, leader_index)
|
||||
if leader_index is None and await option.option.ismulti() and not isinstance(linked_value, list):
|
||||
values = await option.value.get()
|
||||
if linked_value not in values:
|
||||
values.append(linked_value)
|
||||
await option.value.set(values)
|
||||
await option.owner.set('link')
|
||||
else:
|
||||
await option.value.set(linked_value)
|
||||
await option.owner.set('link')
|
||||
except AttributeError as err:
|
||||
if linked_provider == 'oauth2_external':
|
||||
raise ValueError(str(err)) from err
|
||||
pass
|
||||
except Exception as err:
|
||||
await config.property.read_only()
|
||||
raise err from err
|
||||
await config.property.read_only()
|
||||
|
||||
|
||||
def get_ip_from_domain(domain):
|
||||
if not domain:
|
||||
return
|
||||
hostname, domainname = domain.split('.', 1)
|
||||
return DOMAINS[domainname][1][DOMAINS[domainname][0].index(hostname)]
|
||||
|
||||
|
||||
async def load_machine_config(server_name: str,
|
||||
datas: dict,
|
||||
module_info: dict,
|
||||
display_name,
|
||||
):
|
||||
optiondescription = {'set_linked': set_linked,
|
||||
'set_linked_multi_variables': set_linked_multi_variables,
|
||||
'get_linked_configuration': get_linked_configuration,
|
||||
'set_linked_configuration': set_linked_configuration,
|
||||
'get_ip_from_domain': get_ip_from_domain,
|
||||
}
|
||||
cfg = RougailConfig.copy()
|
||||
module_info.servers.append(server_name)
|
||||
cfg['variable_namespace'] = ROUGAIL_NAMESPACE
|
||||
cfg['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
|
||||
if datas['module'] == 'host':
|
||||
cfg['tmpfile_dest_dir'] = datas['values'][f'{ROUGAIL_NAMESPACE}.host_install_dir'] + '/host/configurations/' + server_name
|
||||
cfg['templates_dir'] = module_info.templates_dir
|
||||
cfg['dictionaries_dir'] = module_info.dictionaries_dir
|
||||
cfg['functions_file'] = module_info.functions_file
|
||||
cfg['multi_functions'] = MULTI_FUNCTIONS
|
||||
cfg['extra_dictionaries'] = module_info.extra_dictionaries
|
||||
cfg['extra_annotators'].append('risotto.rougail')
|
||||
cfg['internal_functions'] = list(optiondescription.keys())
|
||||
try:
|
||||
eolobj = RougailConvert(cfg)
|
||||
except Exception as err:
|
||||
print(f'Try to load {module_info.modules}')
|
||||
raise err from err
|
||||
tiram_obj = eolobj.save(None)
|
||||
# if server_name == 'revprox.in.silique.fr':
|
||||
# print(tiram_obj)
|
||||
#cfg['patches_dir'] = join(test_dir, 'patches')
|
||||
try:
|
||||
exec(tiram_obj, None, optiondescription)
|
||||
except Exception as err:
|
||||
print(tiram_obj)
|
||||
raise Exception(f'unknown error when load tiramisu object {err}') from err
|
||||
config = await Config(optiondescription['option_0'], display_name=display_name)
|
||||
await config.property.read_write()
|
||||
try:
|
||||
add_srv = await config.option('machine.add_srv').value.get()
|
||||
except AttributeError:
|
||||
add_srv = False
|
||||
return {'config': config,
|
||||
'templates_informations': cfg,
|
||||
'interface_index': 0,
|
||||
'module_name': datas['module'],
|
||||
'add_srv': add_srv,
|
||||
}
|
||||
|
|
@ -1,6 +1,21 @@
|
|||
from os import environ
|
||||
from json import load
|
||||
from typing import List
|
||||
from ipaddress import ip_address
|
||||
from toml import load as toml_load
|
||||
from pprint import pprint
|
||||
|
||||
|
||||
MULTI_FUNCTIONS = []
|
||||
CONFIGS = {}
|
||||
DOMAINS = {}
|
||||
ZONES = {}
|
||||
ZONES_SERVER = {}
|
||||
SERVERS = {}
|
||||
|
||||
|
||||
with open(environ.get('CONFIG_FILE', 'risotto.conf'), 'r') as fh:
|
||||
RISOTTO_CONFIG = toml_load(fh)
|
||||
|
||||
|
||||
def _(s):
|
||||
|
|
@ -13,3 +28,80 @@ def multi_function(function):
|
|||
if name not in MULTI_FUNCTIONS:
|
||||
MULTI_FUNCTIONS.append(name)
|
||||
return function
|
||||
|
||||
|
||||
async def value_pprint(dico, config):
|
||||
pprint_dict = {}
|
||||
for path, value in dico.items():
|
||||
if await config.option(path).option.type() == 'password' and value:
|
||||
value = 'X' * len(value)
|
||||
pprint_dict[path] = value
|
||||
pprint(pprint_dict)
|
||||
|
||||
|
||||
def load_zones_server():
|
||||
if ZONES_SERVER:
|
||||
return
|
||||
with open('servers.json', 'r') as server_fh:
|
||||
ZONES_SERVER.update(load(server_fh))
|
||||
|
||||
|
||||
def load_zones():
|
||||
global ZONES
|
||||
if ZONES:
|
||||
return
|
||||
|
||||
load_zones_server()
|
||||
ZONES.update(ZONES_SERVER['zones'])
|
||||
for server_name, server in ZONES_SERVER['servers'].items():
|
||||
if 'informations' not in server:
|
||||
continue
|
||||
server_zones = server['informations']['zones_name']
|
||||
server_extra_domainnames = server['informations'].get('extra_domainnames', [])
|
||||
if len(server_zones) > 1 and len(server_zones) != len(server_extra_domainnames) + 1:
|
||||
raise Exception(f'the server "{server_name}" has more that one zone, please set correct number of extra_domainnames ({len(server_zones) - 1} instead of {len(server_extra_domainnames)})')
|
||||
|
||||
for idx, zone_name in enumerate(server_zones):
|
||||
zone_domain_name = ZONES[zone_name]['domain_name']
|
||||
if idx == 0:
|
||||
zone_server_name = server_name
|
||||
else:
|
||||
zone_server_name = server_extra_domainnames[idx - 1]
|
||||
server_domain_name = zone_server_name.split('.', 1)[1]
|
||||
if zone_domain_name and zone_domain_name != server_domain_name:
|
||||
raise Exception(f'wrong server_name "{zone_server_name}" in zone "{zone_name}" should ends with "{zone_domain_name}"')
|
||||
ZONES[zone_name].setdefault('hosts', []).append(server_name)
|
||||
|
||||
|
||||
def load_domains():
|
||||
global DOMAINS
|
||||
if DOMAINS:
|
||||
return
|
||||
load_zones()
|
||||
for zone_name, zone in ZONES_SERVER['zones'].items():
|
||||
if 'domain_name' in zone:
|
||||
hosts = []
|
||||
ips = []
|
||||
for host in ZONES[zone_name].get('hosts', []):
|
||||
hosts.append(host.split('.', 1)[0])
|
||||
ips.append(_get_ip(host, [zone_name], 0))
|
||||
DOMAINS[zone['domain_name']] = (tuple(hosts), tuple(ips))
|
||||
|
||||
|
||||
def _get_ip(server_name: str,
|
||||
zones_name: List[str],
|
||||
index: str,
|
||||
) -> str:
|
||||
if server_name is None or zones_name is None:
|
||||
return
|
||||
load_zones()
|
||||
index = int(index)
|
||||
zone_name = zones_name[index]
|
||||
if zone_name not in ZONES:
|
||||
raise ValueError(f"cannot set IP in unknown zone '{zone_name}'")
|
||||
zone = ZONES[zone_name]
|
||||
if server_name not in zone['hosts']:
|
||||
raise ValueError(f"cannot set IP in unknown server '{server_name}'")
|
||||
server_index = zone['hosts'].index(server_name)
|
||||
# print(server_name, zones_name, index, str(ip_address(zone['start_ip']) + server_index))
|
||||
return str(ip_address(zone['start_ip']) + server_index)
|
||||
|
|
|
|||
Loading…
Reference in a new issue