forked from stove/risotto
533 lines
21 KiB
Python
Executable file
533 lines
21 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
|
|
from asyncio import run
|
|
from os import listdir, link, makedirs, environ
|
|
from os.path import isdir, isfile, join
|
|
from shutil import rmtree, copy2, copytree
|
|
from json import load as json_load
|
|
from yaml import load, SafeLoader
|
|
from toml import load as toml_load
|
|
from pprint import pprint
|
|
from typing import Any
|
|
from warnings import warn_explicit
|
|
from copy import copy
|
|
|
|
from tiramisu import Config
|
|
from tiramisu.error import ValueWarning
|
|
from rougail import RougailConfig, RougailConvert, RougailSystemdTemplate
|
|
from rougail.utils import normalize_family
|
|
|
|
from risotto.utils import MULTI_FUNCTIONS, CONFIGS
|
|
|
|
|
|
with open(environ.get('CONFIG_FILE', 'risotto.conf'), 'r') as fh:
|
|
config = toml_load(fh)
|
|
|
|
|
|
DATASET_DIRECTORY = config['directories']['dataset']
|
|
INSTALL_DIR = config['directories']['dest']
|
|
FUNCTIONS = 'funcs.py'
|
|
CONFIG_DEST_DIR = 'configurations'
|
|
SRV_DEST_DIR = 'srv'
|
|
|
|
|
|
with open('servers.json', 'r') as server_fh:
|
|
jsonfile = json_load(server_fh)
|
|
SERVERS = jsonfile['servers']
|
|
MODULES = jsonfile['modules']
|
|
|
|
|
|
async def set_linked_multi_variables(linked_server: str,
|
|
**kwargs: dict,
|
|
) -> None:
|
|
|
|
if linked_server is None:
|
|
return
|
|
if linked_server not in CONFIGS:
|
|
warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
|
|
ValueWarning,
|
|
__file__,
|
|
0,
|
|
)
|
|
return
|
|
config = CONFIGS[linked_server][0]
|
|
dico = {}
|
|
variables = {}
|
|
for key, value in kwargs.items():
|
|
if value is None:
|
|
return
|
|
if key.startswith('linked_provider_'):
|
|
index = int(key[16])
|
|
path = await config.information.get('provider:' + value, None)
|
|
if not path:
|
|
return
|
|
if index not in variables:
|
|
variables[index] = {'path': None, 'value': None}
|
|
variables[index]['path'] = path
|
|
elif key.startswith('linked_value_'):
|
|
index = int(key[13])
|
|
if index not in variables:
|
|
variables[index] = {'path': None, 'value': None}
|
|
variables[index]['value'] = value
|
|
else:
|
|
raise AttributeError(f'unknown parameter {key}')
|
|
dynamic = None
|
|
slave_idx = None
|
|
await config.property.read_write()
|
|
try:
|
|
for index in sorted(list(variables)):
|
|
path = variables[index]['path']
|
|
value = variables[index]['value']
|
|
if dynamic:
|
|
path = path.replace('{suffix}', dynamic)
|
|
else:
|
|
dynamic = normalize_family(value)
|
|
option = config.forcepermissive.option(path, slave_idx)
|
|
multi = await option.option.ismulti()
|
|
if multi and await option.option.isfollower():
|
|
multi = await option.option.issubmulti()
|
|
if multi:
|
|
values = await option.value.get()
|
|
if value not in values:
|
|
values.append(value)
|
|
await option.value.set(values)
|
|
if await option.option.isleader():
|
|
slave_idx = values.index(value)
|
|
else:
|
|
await option.value.set(value)
|
|
except Exception as err:
|
|
await config.property.read_only()
|
|
raise err from err
|
|
await config.property.read_only()
|
|
|
|
|
|
async def set_linked(linked_server: str,
|
|
linked_provider: str,
|
|
linked_value: str,
|
|
linked_returns: str=None,
|
|
dynamic: str=None,
|
|
):
|
|
if None in (linked_server, linked_provider, linked_value):
|
|
return
|
|
if linked_server not in CONFIGS:
|
|
warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
|
|
ValueWarning,
|
|
__file__,
|
|
0,
|
|
)
|
|
return
|
|
config = CONFIGS[linked_server][0]
|
|
path = await config.information.get('provider:' + linked_provider, None)
|
|
if not path:
|
|
warn_explicit(ValueWarning(f'cannot find provider "{linked_provider}" in linked server "{linked_server}"'),
|
|
ValueWarning,
|
|
__file__,
|
|
0,
|
|
)
|
|
return
|
|
await config.property.read_write()
|
|
try:
|
|
option = config.forcepermissive.option(path)
|
|
if await option.option.ismulti():
|
|
values = await option.value.get()
|
|
if linked_value not in values:
|
|
values.append(linked_value)
|
|
await option.value.set(values)
|
|
else:
|
|
await option.value.set(linked_value)
|
|
except Exception as err:
|
|
await config.property.read_only()
|
|
raise err from err
|
|
await config.property.read_only()
|
|
if linked_returns is not None:
|
|
linked_variable = await config.information.get('provider:' + linked_returns, None)
|
|
if not linked_variable:
|
|
warn_explicit(ValueWarning(f'cannot find linked variable "{linked_returns}" in linked server "{linked_server}"'),
|
|
ValueWarning,
|
|
__file__,
|
|
0,
|
|
)
|
|
return
|
|
else:
|
|
linked_variable = None
|
|
if linked_variable is not None:
|
|
if dynamic:
|
|
linked_variable = linked_variable.replace('{suffix}', normalize_family(dynamic))
|
|
elif '{suffix}' in linked_variable:
|
|
idx = CONFIGS[linked_server][3]
|
|
linked_variable = linked_variable.replace('{suffix}', str(idx))
|
|
ret = await config.forcepermissive.option(linked_variable).value.get()
|
|
else:
|
|
ret = normalize_family(linked_value)
|
|
return ret
|
|
|
|
|
|
async def get_linked_configuration(linked_server: str,
|
|
linked_provider: str,
|
|
dynamic: str=None,
|
|
):
|
|
if linked_server not in CONFIGS:
|
|
warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
|
|
ValueWarning,
|
|
__file__,
|
|
1,
|
|
)
|
|
return
|
|
config = CONFIGS[linked_server][0]
|
|
path = await config.information.get('provider:' + linked_provider, None)
|
|
if not path:
|
|
warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'),
|
|
ValueWarning,
|
|
__file__,
|
|
1,
|
|
)
|
|
return
|
|
if dynamic:
|
|
path = path.replace('{suffix}', normalize_family(dynamic))
|
|
try:
|
|
return await config.forcepermissive.option(path).value.get()
|
|
except AttributeError as err:
|
|
warn_explicit(ValueWarning(f'cannot find get value of "{path}" in linked server "{linked_server}": {err}'),
|
|
ValueWarning,
|
|
__file__,
|
|
1,
|
|
)
|
|
|
|
|
|
class Empty:
|
|
pass
|
|
empty = Empty()
|
|
|
|
|
|
async def set_linked_configuration(_linked_value: Any,
|
|
linked_server: str,
|
|
linked_provider: str,
|
|
linked_value: Any=empty,
|
|
dynamic: str=None,
|
|
leader_provider: str=None,
|
|
leader_value: Any=None,
|
|
leader_index: int=None,
|
|
):
|
|
if linked_value is not empty:
|
|
_linked_value = linked_value
|
|
linked_value = _linked_value
|
|
if linked_server is None:
|
|
return
|
|
if linked_value is None or linked_server not in CONFIGS:
|
|
warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
|
|
ValueWarning,
|
|
__file__,
|
|
2,
|
|
)
|
|
return
|
|
config = CONFIGS[linked_server][0]
|
|
path = await config.information.get('provider:' + linked_provider, None)
|
|
if not path:
|
|
warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'),
|
|
ValueWarning,
|
|
__file__,
|
|
2,
|
|
)
|
|
return
|
|
if dynamic:
|
|
path = path.replace('{suffix}', normalize_family(dynamic))
|
|
await config.property.read_write()
|
|
try:
|
|
if leader_provider is not None:
|
|
leader_path = await config.information.get('provider:' + leader_provider, None)
|
|
if not leader_path:
|
|
await config.property.read_only()
|
|
warn_explicit(ValueWarning(f'cannot find leader variable with leader_provider "{leader_provider}" in linked server "{linked_server}"'),
|
|
ValueWarning,
|
|
__file__,
|
|
2,
|
|
)
|
|
return
|
|
if dynamic:
|
|
leader_path = leader_path.replace('{suffix}', normalize_family(dynamic))
|
|
values = await config.forcepermissive.option(leader_path).value.get()
|
|
if not isinstance(leader_value, list):
|
|
leader_value = [leader_value]
|
|
for lv in leader_value:
|
|
if lv in values:
|
|
slave_idx = values.index(lv)
|
|
slave_option = config.forcepermissive.option(path, slave_idx)
|
|
if await slave_option.option.issubmulti():
|
|
slave_values = await slave_option.value.get()
|
|
if linked_value not in slave_values:
|
|
slave_values.append(linked_value)
|
|
await slave_option.value.set(slave_values)
|
|
else:
|
|
await slave_option.value.set(linked_value)
|
|
else:
|
|
option = config.forcepermissive.option(path, leader_index)
|
|
if leader_index is None and await option.option.ismulti() and not isinstance(linked_value, list):
|
|
values = await option.value.get()
|
|
if linked_value not in values:
|
|
values.append(linked_value)
|
|
await option.value.set(values)
|
|
else:
|
|
await option.value.set(linked_value)
|
|
except AttributeError as err:
|
|
#raise ValueError(str(err)) from err
|
|
pass
|
|
except Exception as err:
|
|
await config.property.read_only()
|
|
raise err from err
|
|
await config.property.read_only()
|
|
|
|
|
|
def tiramisu_display_name(kls,
|
|
dyn_name: 'Base'=None,
|
|
suffix: str=None,
|
|
) -> str:
|
|
if dyn_name is not None:
|
|
name = kls.impl_getpath() + suffix
|
|
else:
|
|
name = kls.impl_getpath()
|
|
return name
|
|
|
|
|
|
def load_applications():
|
|
applications = {}
|
|
distrib_dir = join(DATASET_DIRECTORY, 'applicationservice')
|
|
for release in listdir(distrib_dir):
|
|
release_dir = join(distrib_dir, release)
|
|
if not isdir(release_dir):
|
|
continue
|
|
for applicationservice in listdir(release_dir):
|
|
applicationservice_dir = join(release_dir, applicationservice)
|
|
if not isdir(applicationservice_dir):
|
|
continue
|
|
if applicationservice in applications:
|
|
raise Exception(f'multi applicationservice: {applicationservice} ({applicationservice_dir} <=> {applications[applicationservice]})')
|
|
applications[applicationservice] = applicationservice_dir
|
|
return applications
|
|
|
|
|
|
class ModuleCfg():
|
|
def __init__(self):
|
|
self.dictionaries_dir = []
|
|
self.modules = []
|
|
self.functions_file = [FUNCTIONS]
|
|
self.templates_dir = []
|
|
self.extra_dictionaries = {}
|
|
self.servers = []
|
|
|
|
|
|
def build_module(module_name, datas, module_infos):
|
|
install_dir = join(INSTALL_DIR, module_name)
|
|
makedirs(install_dir)
|
|
applications = load_applications()
|
|
cfg = ModuleCfg()
|
|
module_infos[module_name] = cfg
|
|
def calc_depends(appname, added):
|
|
if appname in added:
|
|
return
|
|
as_dir = applications[appname]
|
|
cfg.modules.append(appname)
|
|
dictionaries_dir = join(as_dir, 'dictionaries')
|
|
if isdir(dictionaries_dir):
|
|
cfg.dictionaries_dir.append(dictionaries_dir)
|
|
funcs_dir = join(as_dir, 'funcs')
|
|
if isdir(funcs_dir):
|
|
for f in listdir(funcs_dir):
|
|
if f.startswith('__'):
|
|
continue
|
|
cfg.functions_file.append(join(funcs_dir, f))
|
|
templates_dir = join(as_dir, 'templates')
|
|
if isdir(templates_dir):
|
|
cfg.templates_dir.append(templates_dir)
|
|
extras_dir = join(as_dir, 'extras')
|
|
if isdir(extras_dir):
|
|
for extra in listdir(extras_dir):
|
|
extra_dir = join(extras_dir, extra)
|
|
if isdir(extra_dir):
|
|
cfg.extra_dictionaries.setdefault(extra, []).append(extra_dir)
|
|
for type in ['image', 'install']:
|
|
manual_dir = join(as_dir, 'manual', type)
|
|
if isdir(manual_dir):
|
|
for filename in listdir(manual_dir):
|
|
src_file = join(manual_dir, filename)
|
|
if type == 'image':
|
|
dst_file = join(install_dir, filename)
|
|
verify = False
|
|
else:
|
|
dst_file= join(INSTALL_DIR, filename)
|
|
verify = True
|
|
if isdir(src_file):
|
|
if not isdir(dst_file):
|
|
makedirs(dst_file)
|
|
for subfilename in listdir(src_file):
|
|
if not verify or not isfile(dst_file):
|
|
src = join(src_file, subfilename)
|
|
dst = join(dst_file, subfilename)
|
|
if isfile(src):
|
|
copy2(src, dst)
|
|
else:
|
|
copytree(src, dst)
|
|
elif not verify or not isfile(dst_file):
|
|
src = join(manual_dir, filename)
|
|
dst = dst_file
|
|
if isfile(src):
|
|
copy2(src, dst)
|
|
else:
|
|
copytree(src, dst)
|
|
added.append(appname)
|
|
with open(join(as_dir, 'applicationservice.yml')) as yaml:
|
|
app = load(yaml, Loader=SafeLoader)
|
|
|
|
for xml in app.get('depends', []):
|
|
calc_depends(xml, added)
|
|
added = []
|
|
for applicationservice in datas['applicationservices']:
|
|
calc_depends(applicationservice, added)
|
|
|
|
|
|
async def build(server_name, datas, module_infos):
|
|
if server_name in CONFIGS:
|
|
raise Exception(f'server "{server_name}" is duplicate')
|
|
cfg = RougailConfig.copy()
|
|
module_info = module_infos[datas['module']]
|
|
module_info.servers.append(server_name)
|
|
if datas['module'] == 'host':
|
|
cfg['tmpfile_dest_dir'] = datas['values']['rougail.host_install_dir'] + '/host/configurations/' + server_name
|
|
cfg['templates_dir'] = module_info.templates_dir
|
|
cfg['dictionaries_dir'] = module_info.dictionaries_dir
|
|
cfg['functions_file'] = module_info.functions_file
|
|
cfg['multi_functions'] = MULTI_FUNCTIONS
|
|
cfg['extra_dictionaries'] = module_info.extra_dictionaries
|
|
cfg['extra_annotators'].append('risotto.rougail')
|
|
optiondescription = {'set_linked': set_linked,
|
|
'set_linked_multi_variables': set_linked_multi_variables,
|
|
'get_linked_configuration': get_linked_configuration,
|
|
'set_linked_configuration': set_linked_configuration,
|
|
}
|
|
cfg['internal_functions'] = list(optiondescription.keys())
|
|
try:
|
|
eolobj = RougailConvert(cfg)
|
|
except Exception as err:
|
|
print(f'Try to load {module_info.modules}')
|
|
raise err from err
|
|
tiram_obj = eolobj.save(None)
|
|
# if server_name == 'revprox.in.silique.fr':
|
|
# print(tiram_obj)
|
|
#cfg['patches_dir'] = join(test_dir, 'patches')
|
|
cfg['tmp_dir'] = 'tmp'
|
|
cfg['destinations_dir'] = join(INSTALL_DIR, datas['module'], CONFIG_DEST_DIR, server_name)
|
|
if isdir('tmp'):
|
|
rmtree('tmp')
|
|
makedirs('tmp')
|
|
makedirs(cfg['destinations_dir'])
|
|
try:
|
|
exec(tiram_obj, None, optiondescription)
|
|
except Exception as err:
|
|
print(tiram_obj)
|
|
raise Exception(f'unknown error when load tiramisu object {err}') from err
|
|
config = await Config(optiondescription['option_0'], display_name=tiramisu_display_name)
|
|
await config.property.read_write()
|
|
try:
|
|
if await config.option('machine.add_srv').value.get():
|
|
srv = join(INSTALL_DIR, SRV_DEST_DIR, server_name)
|
|
else:
|
|
srv = None
|
|
except AttributeError:
|
|
srv = None
|
|
await config.property.read_write()
|
|
CONFIGS[server_name] = (config, cfg, srv, 0)
|
|
|
|
|
|
async def value_pprint(dico, config):
|
|
pprint_dict = {}
|
|
for path, value in dico.items():
|
|
if await config.option(path).option.type() == 'password' and value:
|
|
value = 'X' * len(value)
|
|
pprint_dict[path] = value
|
|
pprint(pprint_dict)
|
|
|
|
|
|
async def set_values(server_name, config, datas):
|
|
if 'informations' in datas:
|
|
for information, value in datas['informations'].items():
|
|
await config.information.set(information, value)
|
|
if 'extra_domainnames' in datas['informations']:
|
|
for idx, extra_domainname in enumerate(datas['informations']['extra_domainnames']):
|
|
if extra_domainname in CONFIGS:
|
|
raise Exception(f'server "{server_name}" is duplicate')
|
|
value = list(CONFIGS[server_name])
|
|
value[3] = idx + 1
|
|
CONFIGS[extra_domainname] = tuple(value)
|
|
await config.information.set('server_name', server_name)
|
|
await config.property.read_write()
|
|
try:
|
|
if 'values' in datas:
|
|
for path, value in datas['values'].items():
|
|
if isinstance(value, dict):
|
|
for idx, val in value.items():
|
|
await config.option(path, int(idx)).value.set(val)
|
|
else:
|
|
await config.option(path).value.set(value)
|
|
except Exception as err:
|
|
await value_pprint(await config.value.dict(), config)
|
|
error_msg = f'cannot configure server "{server_name}": {err}'
|
|
raise Exception(error_msg) from err
|
|
await config.property.read_only()
|
|
#await config.value.dict()
|
|
|
|
|
|
async def valid_mandatories(server_name, config):
|
|
mandatories = await config.value.mandatory()
|
|
if mandatories:
|
|
print()
|
|
print(f'=== Configuration: {server_name} ===')
|
|
await config.property.pop('mandatory')
|
|
await value_pprint(await config.value.dict(), config)
|
|
raise Exception(f'server "{server_name}" has mandatories variables without values "{", ".join(mandatories)}"')
|
|
|
|
|
|
async def templates(server_name, config, cfg, srv, int_idx):
|
|
values = await config.value.dict()
|
|
engine = RougailSystemdTemplate(config, cfg)
|
|
# if server_name == 'revprox.in.silique.fr':
|
|
# print()
|
|
# print(f'=== Configuration: {server_name} ===')
|
|
# pprint(values)
|
|
try:
|
|
await engine.instance_files()
|
|
except Exception as err:
|
|
print()
|
|
print(f'=== Configuration: {server_name} ===')
|
|
await value_pprint(values, config)
|
|
raise err from err
|
|
if srv:
|
|
makedirs(srv)
|
|
|
|
|
|
async def main():
|
|
if isdir(INSTALL_DIR):
|
|
rmtree(INSTALL_DIR)
|
|
makedirs(INSTALL_DIR)
|
|
module_infos = {}
|
|
for module_name, datas in MODULES.items():
|
|
build_module(module_name, datas, module_infos)
|
|
for server_name, datas in SERVERS.items():
|
|
await build(server_name, datas, module_infos)
|
|
for module_name, cfg in module_infos.items():
|
|
with open(join(INSTALL_DIR, module_name, 'install_machines'), 'w') as fh:
|
|
for server_name in cfg.servers:
|
|
fh.write(f'./install_machine {module_name} {server_name}\n')
|
|
for server_name, datas in SERVERS.items():
|
|
await set_values(server_name, CONFIGS[server_name][0], datas)
|
|
for server_name in SERVERS:
|
|
config = CONFIGS[server_name][0]
|
|
await config.property.pop('mandatory')
|
|
await config.value.dict()
|
|
await config.property.add('mandatory')
|
|
for server_name in SERVERS:
|
|
await valid_mandatories(server_name, CONFIGS[server_name][0])
|
|
# print(await CONFIGS['revprox.in.gnunux.info'][0].option('nginx.reverse_proxy_for_netbox_in_gnunux_info.reverse_proxy_netbox_in_gnunux_info.revprox_url_netbox_in_gnunux_info', 0).value.get())
|
|
for server_name in SERVERS:
|
|
await templates(server_name, *CONFIGS[server_name])
|
|
|
|
|
|
run(main())
|