forked from stove/risotto
404 lines
18 KiB
Python
404 lines
18 KiB
Python
from .utils import MULTI_FUNCTIONS, load_zones, value_pprint, RISOTTO_CONFIG, EXTRA_ANNOTATORS, ROUGAIL_NAMESPACE, ROUGAIL_NAMESPACE_DESCRIPTION
|
|
from .image import Applications, Modules, valid_mandatories, applicationservice_copy
|
|
from .rougail.annotator import calc_providers, calc_providers_global, calc_providers_dynamic, calc_providers_dynamic_follower, calc_providers_follower
|
|
|
|
from rougail import RougailConfig, RougailConvert
|
|
from os import remove, makedirs, listdir, chmod
|
|
from os.path import isfile, isdir, abspath, join, dirname
|
|
from json import dump as json_dump, load as json_load
|
|
from yaml import load as yaml_load, SafeLoader
|
|
#
|
|
from tiramisu import Config, valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal, calc_value
|
|
from rougail.utils import normalize_family
|
|
from rougail import RougailSystemdTemplate
|
|
from shutil import copy2, copytree, rmtree
|
|
|
|
|
|
def tiramisu_display_name(kls,
|
|
dyn_name: 'Base'=None,
|
|
suffix: str=None,
|
|
) -> str:
|
|
# FIXME
|
|
if dyn_name is not None:
|
|
name = kls.impl_getpath() + str(suffix)
|
|
else:
|
|
name = kls.impl_getpath()
|
|
return name
|
|
|
|
|
|
CONFIG_FILE = 'servers.yml'
|
|
TIRAMISU_CACHE = 'tiramisu_cache.py'
|
|
VALUES_CACHE = 'values_cache.json'
|
|
INFORMATIONS_CACHE = 'informations_cache.json'
|
|
INSTALL_DIR = RISOTTO_CONFIG['directories']['dest']
|
|
INSTALL_CONFIG_DIR = 'configurations'
|
|
INSTALL_TMPL_DIR= 'templates'
|
|
INSTALL_IMAGES_DIR = 'images_files'
|
|
INSTALL_TESTS_DIR = 'tests'
|
|
FUNCTIONS = {'calc_providers': calc_providers,
|
|
'calc_providers_global': calc_providers_global,
|
|
'calc_providers_dynamic': calc_providers_dynamic,
|
|
'calc_providers_dynamic_follower': calc_providers_dynamic_follower,
|
|
'calc_providers_follower': calc_providers_follower,
|
|
'valid_network_netmask': valid_network_netmask,
|
|
'valid_ip_netmask': valid_ip_netmask,
|
|
'valid_broadcast': valid_broadcast,
|
|
'valid_in_network': valid_in_network,
|
|
'valid_not_equal': valid_not_equal,
|
|
'calc_value': calc_value,
|
|
'normalize_family': normalize_family,
|
|
}
|
|
|
|
|
|
def copy(src_file, dst_file):
|
|
if isdir(src_file):
|
|
if not isdir(dst_file):
|
|
makedirs(dst_file)
|
|
for subfilename in listdir(src_file):
|
|
if not isfile(dst_file):
|
|
src = join(src_file, subfilename)
|
|
dst = join(dst_file, subfilename)
|
|
if isfile(src):
|
|
copy2(src, dst)
|
|
else:
|
|
copytree(src, dst)
|
|
elif not isfile(dst_file):
|
|
dst = dirname(dst_file)
|
|
if not isdir(dst):
|
|
makedirs(dst)
|
|
if isfile(src_file):
|
|
copy2(src_file, dst_file)
|
|
else:
|
|
copytree(src_file, dst_file)
|
|
|
|
|
|
def re_create(dir_name):
|
|
if isdir(dir_name):
|
|
rmtree(dir_name)
|
|
makedirs(dir_name)
|
|
|
|
|
|
def remove_cache():
|
|
if isfile(TIRAMISU_CACHE):
|
|
remove(TIRAMISU_CACHE)
|
|
if isfile(VALUES_CACHE):
|
|
remove(VALUES_CACHE)
|
|
if isfile(INFORMATIONS_CACHE):
|
|
remove(INFORMATIONS_CACHE)
|
|
|
|
|
|
async def templates(server_name,
|
|
config,
|
|
just_copy=False,
|
|
copy_manuals=False,
|
|
template=None,
|
|
):
|
|
subconfig = config.option(normalize_family(server_name))
|
|
try:
|
|
await subconfig.option.get()
|
|
except:
|
|
servers = [await server.option.description() for server in await config.option.list('optiondescription')]
|
|
raise Exception(f'cannot find server name "{server_name}": {servers}')
|
|
|
|
rougailconfig = RougailConfig.copy()
|
|
rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE
|
|
rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
|
|
rougailconfig['tmp_dir'] = 'tmp'
|
|
rougailconfig['templates_dir'] = await subconfig.information.get('templates_dir')
|
|
rougailconfig['patches_dir'] = await subconfig.information.get('patches_dir')
|
|
rougailconfig['functions_file'] = await subconfig.information.get('functions_files')
|
|
module = await subconfig.information.get('module')
|
|
is_host = module == 'host'
|
|
if is_host:
|
|
rougailconfig['systemd_tmpfile_delete_before_create'] = True
|
|
if just_copy:
|
|
raise Exception('cannot generate template with option just_copy for a host')
|
|
else:
|
|
rougailconfig['systemd_tmpfile_delete_before_create'] = False
|
|
#rougailconfig['systemd_tmpfile_factory_dir'] = '/usr/local/lib'
|
|
if not just_copy:
|
|
rougailconfig['destinations_dir'] = join(INSTALL_DIR, INSTALL_CONFIG_DIR, server_name)
|
|
else:
|
|
rougailconfig['destinations_dir'] = join(INSTALL_DIR, INSTALL_TMPL_DIR, server_name)
|
|
re_create(rougailconfig['destinations_dir'])
|
|
re_create(rougailconfig['tmp_dir'])
|
|
|
|
engine = RougailSystemdTemplate(subconfig, rougailconfig)
|
|
if just_copy:
|
|
# for all engine to none
|
|
ori_engines = {}
|
|
for eng in engine.engines:
|
|
if eng == 'none':
|
|
continue
|
|
ori_engines[eng] = engine.engines[eng]
|
|
engine.engines[eng] = engine.engines['none']
|
|
try:
|
|
if not template:
|
|
await engine.instance_files()
|
|
else:
|
|
await engine.instance_file(template)
|
|
except Exception as err:
|
|
print()
|
|
print(f'=== Configuration: {server_name} ===')
|
|
values = await subconfig.value.dict()
|
|
await value_pprint(values, subconfig)
|
|
raise err from err
|
|
if just_copy:
|
|
for eng, old_engine in ori_engines.items():
|
|
engine.engines[eng] = old_engine
|
|
secrets_dir = join(rougailconfig['destinations_dir'], 'secrets')
|
|
if isdir(secrets_dir):
|
|
chmod(secrets_dir, 0o700)
|
|
if copy_manuals and not is_host:
|
|
dest_dir = join(INSTALL_DIR, INSTALL_IMAGES_DIR, module)
|
|
if not isdir(dest_dir):
|
|
for manual in await subconfig.information.get('manuals_dirs'):
|
|
for filename in listdir(manual):
|
|
src_file = join(manual, filename)
|
|
dst_file = join(dest_dir, filename)
|
|
copy(src_file, dst_file)
|
|
copy_tests = await config.information.get('copy_tests')
|
|
|
|
if copy_tests and not is_host:
|
|
dest_dir = join(INSTALL_DIR, INSTALL_TESTS_DIR, module)
|
|
if not isdir(dest_dir):
|
|
for tests in await subconfig.information.get('tests_dirs'):
|
|
for filename in listdir(tests):
|
|
src_file = join(tests, filename)
|
|
dst_file = join(dest_dir, filename)
|
|
copy(src_file, dst_file)
|
|
|
|
|
|
class Loader:
|
|
def __init__(self,
|
|
clean_directories,
|
|
hide_secret,
|
|
original_display_name,
|
|
valid_mandatories,
|
|
config_file=CONFIG_FILE,
|
|
):
|
|
self.hide_secret = hide_secret
|
|
self.original_display_name = original_display_name
|
|
self.valid_mandatories = valid_mandatories
|
|
self.config_file = config_file
|
|
if clean_directories:
|
|
if isdir(INSTALL_DIR):
|
|
rmtree(INSTALL_DIR)
|
|
makedirs(INSTALL_DIR)
|
|
|
|
def load_tiramisu_file(self):
|
|
"""Load config file (servers.yml) and build tiramisu file with dataset informations
|
|
"""
|
|
with open(self.config_file, 'r') as server_fh:
|
|
self.servers_json = yaml_load(server_fh, Loader=SafeLoader)
|
|
# set global rougail configuration
|
|
cfg = RougailConfig.copy()
|
|
cfg['variable_namespace'] = ROUGAIL_NAMESPACE
|
|
cfg['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
|
|
cfg['multi_functions'] = MULTI_FUNCTIONS
|
|
cfg['extra_annotators'] = EXTRA_ANNOTATORS
|
|
cfg['internal_functions'] = list(FUNCTIONS.keys())
|
|
cfg['force_convert_dyn_option_description'] = True
|
|
cfg['risotto_globals'] = {}
|
|
|
|
# initialise variables to store useful informations
|
|
# those variables are use during templating
|
|
self.templates_dir = {}
|
|
self.patches_dir = {}
|
|
self.functions_files = {}
|
|
self.manuals_dirs = {}
|
|
self.tests_dirs = {}
|
|
self.modules = {}
|
|
|
|
functions_files = set()
|
|
applicationservices = Applications()
|
|
zones = self.servers_json['zones']
|
|
|
|
rougail = RougailConvert(cfg)
|
|
for host_name, datas in self.servers_json['hosts'].items():
|
|
# load modules associate to this host
|
|
modules_name = set()
|
|
for name, mod_datas in datas['servers'].items():
|
|
if not 'module' in mod_datas:
|
|
raise Exception(f'module is mandatory for "{name}"')
|
|
modules_name.add(mod_datas['module'])
|
|
# load modules informations from config files
|
|
modules = Modules(datas['applicationservices'],
|
|
applicationservices,
|
|
datas['applicationservice_provider'],
|
|
modules_name,
|
|
self.servers_json['modules']
|
|
)
|
|
|
|
# load host
|
|
module_info = modules.get('host')
|
|
cfg['risotto_globals'][host_name] = {'global:server_name': host_name,
|
|
'global:module_name': 'host',
|
|
'global:host_install_dir': abspath(INSTALL_DIR),
|
|
}
|
|
functions_files |= set(module_info.functions_file)
|
|
self.load_dictionaries(cfg,
|
|
module_info,
|
|
host_name,
|
|
rougail,
|
|
)
|
|
# load servers
|
|
modules_info = {}
|
|
for server_name, server_datas in datas['servers'].items():
|
|
module_info = modules.get(server_datas['module'])
|
|
zones_name = server_datas['informations']['zones_name']
|
|
values = [f'{server_name}.{zones[zone_name]["domain_name"]}' for zone_name in zones_name]
|
|
cfg['risotto_globals'][values[0]] = {'global:host_name': host_name,
|
|
'global:server_name': values[0],
|
|
'global:server_names': values,
|
|
'global:zones_name': zones_name,
|
|
'global:zones_list': list(range(len(zones_name))),
|
|
'global:module_name': server_datas['module'],
|
|
}
|
|
server_datas['server_name'] = values[0]
|
|
functions_files |= set(module_info.functions_file)
|
|
self.load_dictionaries(cfg,
|
|
module_info,
|
|
values[0],
|
|
rougail,
|
|
)
|
|
modules_info[module_info.module_name] = module_info.depends
|
|
self.modules[host_name] = modules_info
|
|
cfg['functions_file'] = list(functions_files)
|
|
self.tiram_obj = rougail.save(TIRAMISU_CACHE)
|
|
|
|
def load_dictionaries(self, cfg, module_info, server_name, rougail):
|
|
cfg['dictionaries_dir'] = module_info.dictionaries_dir
|
|
cfg['extra_dictionaries'] = module_info.extra_dictionaries
|
|
cfg['functions_file'] = module_info.functions_file
|
|
rougail.load_dictionaries(path_prefix=server_name)
|
|
self.templates_dir[server_name] = module_info.templates_dir
|
|
self.patches_dir[server_name] = module_info.patches_dir
|
|
self.functions_files[server_name] = module_info.functions_file
|
|
self.manuals_dirs[server_name] = module_info.manuals
|
|
self.tests_dirs[server_name] = module_info.tests
|
|
|
|
async def tiramisu_file_to_tiramisu(self):
|
|
# l
|
|
tiramisu_space = FUNCTIONS.copy()
|
|
try:
|
|
exec(self.tiram_obj, None, tiramisu_space)
|
|
except Exception as err:
|
|
print(self.tiram_obj)
|
|
raise Exception(f'unknown error when load tiramisu object {err}') from err
|
|
if self.original_display_name:
|
|
display_name = None
|
|
else:
|
|
display_name = tiramisu_display_name
|
|
self.config = await Config(tiramisu_space['option_0'],
|
|
display_name=display_name,
|
|
)
|
|
|
|
async def load_values_and_informations(self):
|
|
config = self.config
|
|
await config.property.read_write()
|
|
await config.property.pop('validator')
|
|
await config.property.pop('cache')
|
|
load_zones(self.servers_json)
|
|
await config.information.set('zones', self.servers_json['zones'])
|
|
for host_name, hosts_datas in self.servers_json['hosts'].items():
|
|
information = config.option(normalize_family(host_name)).information
|
|
await information.set('module', 'host')
|
|
await information.set('templates_dir', self.templates_dir[host_name])
|
|
await information.set('patches_dir', self.patches_dir[host_name])
|
|
await information.set('functions_files', self.functions_files[host_name])
|
|
await self.set_values(host_name, config, hosts_datas)
|
|
for datas in hosts_datas['servers'].values():
|
|
server_name = datas['server_name']
|
|
information = config.option(normalize_family(server_name)).information
|
|
await information.set('module', datas['module'])
|
|
await information.set('templates_dir', self.templates_dir[server_name])
|
|
await information.set('patches_dir', self.patches_dir[server_name])
|
|
await information.set('functions_files', self.functions_files[server_name])
|
|
await information.set('manuals_dirs', self.manuals_dirs[server_name])
|
|
await information.set('tests_dirs', self.tests_dirs[server_name])
|
|
await self.set_values(server_name, config, datas)
|
|
|
|
await config.information.set('copy_tests', False)
|
|
# FIXME only one host_name is supported
|
|
await config.information.set('modules', self.modules[host_name])
|
|
# await config.information.set('modules', {module_name: module_info.depends for module_name, module_info in self.module_infos.items() if module_name in modules})
|
|
await config.property.add('cache')
|
|
if self.valid_mandatories:
|
|
messages = await valid_mandatories(config)
|
|
if messages:
|
|
msg = ''
|
|
for title, variables in messages.items():
|
|
msg += '\n' + title + '\n'
|
|
msg += '\n'.join(variables)
|
|
raise Exception(msg)
|
|
await config.property.read_only()
|
|
with open(VALUES_CACHE, 'w') as fh:
|
|
json_dump(await config.value.exportation(), fh)
|
|
with open(INFORMATIONS_CACHE, 'w') as fh:
|
|
json_dump(await config.information.exportation(), fh)
|
|
|
|
async def set_values(self,
|
|
server_name,
|
|
config,
|
|
datas,
|
|
):
|
|
if 'values' not in datas:
|
|
return
|
|
if not isinstance(datas['values'], dict):
|
|
raise Exception(f'Values of "{server_name}" are not a dict: {datas["values"]}')
|
|
server_path = normalize_family(server_name)
|
|
await config.owner.set(self.config_file)
|
|
for vpath, value in datas['values'].items():
|
|
path = f'{server_path}.{vpath}'
|
|
try:
|
|
if isinstance(value, dict):
|
|
for idx, val in value.items():
|
|
await config.option(path, int(idx)).value.set(val)
|
|
else:
|
|
await config.option(path).value.set(value)
|
|
except Exception as err:
|
|
await value_pprint(await config.value.dict(), config)
|
|
error_msg = f'cannot configure variable {vpath} for server "{server_name}": {err}'
|
|
raise Exception(error_msg) from err
|
|
await config.owner.set('user')
|
|
|
|
|
|
class LoaderCache(Loader):
|
|
def load_tiramisu_file(self):
|
|
with open(TIRAMISU_CACHE) as fh:
|
|
self.tiram_obj = fh.read()
|
|
|
|
async def load_values_and_informations(self):
|
|
with open(VALUES_CACHE, 'r') as fh:
|
|
await self.config.value.importation(json_load(fh))
|
|
with open(INFORMATIONS_CACHE, 'r') as fh:
|
|
informations = json_load(fh)
|
|
# null is not a valid key in json => 'null'
|
|
informations[None] = informations.pop('null')
|
|
await self.config.information.importation(informations)
|
|
|
|
|
|
async def load(clean_directories=False,
|
|
hide_secret=False,
|
|
original_display_name: bool=False,
|
|
valid_mandatories: bool=True,
|
|
copy_tests: bool=False,
|
|
):
|
|
if isfile(TIRAMISU_CACHE) and isfile(VALUES_CACHE) and isfile(INFORMATIONS_CACHE):
|
|
loader_obj = LoaderCache
|
|
else:
|
|
loader_obj = Loader
|
|
loader = loader_obj(clean_directories,
|
|
hide_secret,
|
|
original_display_name,
|
|
valid_mandatories,
|
|
)
|
|
loader.load_tiramisu_file()
|
|
await loader.tiramisu_file_to_tiramisu()
|
|
await loader.load_values_and_informations()
|
|
config = loader.config
|
|
await config.property.read_only()
|
|
await config.information.set('copy_tests', copy_tests)
|
|
await config.cache.reset()
|
|
return config
|