risotto/src/risotto/machine.py

585 lines
27 KiB
Python

from .utils import MULTI_FUNCTIONS, load_zones, value_pprint, RISOTTO_CONFIG, EXTRA_ANNOTATORS, ROUGAIL_NAMESPACE, ROUGAIL_NAMESPACE_DESCRIPTION
from .image import Applications, Modules, valid_mandatories, applicationservice_copy
from .rougail.annotator import calc_providers, calc_providers_global, calc_providers_dynamic, calc_providers_dynamic_follower, calc_providers_follower
from rougail import RougailConfig, RougailConvert
from os import remove, makedirs, listdir, chmod
from os.path import isfile, isdir, abspath, join, dirname
from json import dump as json_dump, load as json_load
from yaml import load as yaml_load, SafeLoader
from ipaddress import ip_network
#
from tiramisu import Config, valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal, calc_value
from rougail.utils import normalize_family
from rougail import RougailSystemdTemplate
from shutil import copy2, copytree, rmtree
def tiramisu_display_name(kls,
dyn_name: 'Base'=None,
suffix: str=None,
) -> str:
# FIXME
if dyn_name is not None:
name = kls.impl_getpath() + str(suffix)
else:
name = kls.impl_getpath()
return name
CONFIG_FILE = 'servers.yml'
TIRAMISU_CACHE = 'tiramisu_cache.py'
VALUES_CACHE = 'values_cache.json'
INFORMATIONS_CACHE = 'informations_cache.json'
INSTALL_DIR = RISOTTO_CONFIG['directories']['dest']
INSTALL_CONFIG_DIR = 'configurations'
INSTALL_TMPL_DIR= 'templates'
INSTALL_IMAGES_DIR = 'images_files'
INSTALL_TESTS_DIR = 'tests'
FUNCTIONS = {'calc_providers': calc_providers,
'calc_providers_global': calc_providers_global,
'calc_providers_dynamic': calc_providers_dynamic,
'calc_providers_dynamic_follower': calc_providers_dynamic_follower,
'calc_providers_follower': calc_providers_follower,
'valid_network_netmask': valid_network_netmask,
'valid_ip_netmask': valid_ip_netmask,
'valid_broadcast': valid_broadcast,
'valid_in_network': valid_in_network,
'valid_not_equal': valid_not_equal,
'calc_value': calc_value,
'normalize_family': normalize_family,
}
def copy(src_file, dst_file):
if isdir(src_file):
if not isdir(dst_file):
makedirs(dst_file)
for subfilename in listdir(src_file):
if not isfile(dst_file):
src = join(src_file, subfilename)
dst = join(dst_file, subfilename)
if isfile(src):
copy2(src, dst)
else:
copytree(src, dst)
elif not isfile(dst_file):
dst = dirname(dst_file)
if not isdir(dst):
makedirs(dst)
if isfile(src_file):
copy2(src_file, dst_file)
else:
copytree(src_file, dst_file)
def re_create(dir_name):
if isdir(dir_name):
rmtree(dir_name)
makedirs(dir_name)
def remove_cache():
if isfile(TIRAMISU_CACHE):
remove(TIRAMISU_CACHE)
if isfile(VALUES_CACHE):
remove(VALUES_CACHE)
if isfile(INFORMATIONS_CACHE):
remove(INFORMATIONS_CACHE)
async def templates(server_name,
config,
just_copy=False,
copy_manuals=False,
template=None,
extra_variables=None,
):
subconfig = config.option(normalize_family(server_name))
try:
await subconfig.option.get()
except:
servers = [await server.option.description() for server in await config.option.list('optiondescription')]
raise Exception(f'cannot find server name "{server_name}": {servers}')
rougailconfig = RougailConfig.copy()
rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE
rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
rougailconfig['tmp_dir'] = 'tmp'
rougailconfig['templates_dir'] = await subconfig.information.get('templates_dir')
rougailconfig['patches_dir'] = await subconfig.information.get('patches_dir')
rougailconfig['functions_file'] = await subconfig.information.get('functions_files')
module = await subconfig.information.get('module')
is_host = module == 'host'
if is_host:
rougailconfig['systemd_tmpfile_delete_before_create'] = True
if just_copy:
raise Exception('cannot generate template with option just_copy for a host')
else:
rougailconfig['systemd_tmpfile_delete_before_create'] = False
#rougailconfig['systemd_tmpfile_factory_dir'] = '/usr/local/lib'
if not just_copy:
rougailconfig['destinations_dir'] = join(INSTALL_DIR, INSTALL_CONFIG_DIR, server_name)
else:
rougailconfig['destinations_dir'] = join(INSTALL_DIR, INSTALL_TMPL_DIR, server_name)
re_create(rougailconfig['destinations_dir'])
re_create(rougailconfig['tmp_dir'])
engine = RougailSystemdTemplate(subconfig,
rougailconfig,
)
if just_copy:
# for all engine to none
ori_engines = {}
for eng in engine.engines:
if eng == 'none':
continue
ori_engines[eng] = engine.engines[eng]
engine.engines[eng] = engine.engines['none']
try:
if not template:
await engine.instance_files(extra_variables=extra_variables)
else:
await engine.instance_file(template, extra_variables=extra_variables)
except Exception as err:
print()
print(f'=== Configuration: {server_name} ===')
try:
values = await subconfig.value.dict()
await value_pprint(values, subconfig)
except:
pass
raise err from err
if just_copy:
for eng, old_engine in ori_engines.items():
engine.engines[eng] = old_engine
secrets_dir = join(rougailconfig['destinations_dir'], 'secrets')
if isdir(secrets_dir):
chmod(secrets_dir, 0o700)
if copy_manuals and not is_host:
dest_dir = join(INSTALL_DIR, INSTALL_IMAGES_DIR, module)
if not isdir(dest_dir):
for manual in await subconfig.information.get('manuals_dirs'):
for filename in listdir(manual):
src_file = join(manual, filename)
dst_file = join(dest_dir, filename)
copy(src_file, dst_file)
copy_tests = await config.information.get('copy_tests')
if copy_tests and not is_host:
dest_dir = join(INSTALL_DIR, INSTALL_TESTS_DIR, module)
if not isdir(dest_dir):
for tests in await subconfig.information.get('tests_dirs'):
for filename in listdir(tests):
src_file = join(tests, filename)
dst_file = join(dest_dir, filename)
copy(src_file, dst_file)
class Loader:
def __init__(self,
clean_directories,
hide_secret,
original_display_name,
valid_mandatories,
config_file=CONFIG_FILE,
):
self.hide_secret = hide_secret
self.original_display_name = original_display_name
self.valid_mandatories = valid_mandatories
self.config_file = config_file
if clean_directories:
if isdir(INSTALL_DIR):
rmtree(INSTALL_DIR)
makedirs(INSTALL_DIR)
def load_tiramisu_file(self):
"""Load config file (servers.yml) and build tiramisu file with dataset informations
"""
with open(self.config_file, 'r') as server_fh:
self.servers_json = yaml_load(server_fh, Loader=SafeLoader)
self.add_tls()
# set global rougail configuration
cfg = RougailConfig.copy()
cfg['variable_namespace'] = ROUGAIL_NAMESPACE
cfg['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
cfg['multi_functions'] = MULTI_FUNCTIONS
cfg['extra_annotators'] = EXTRA_ANNOTATORS
cfg['internal_functions'] = list(FUNCTIONS.keys())
cfg['force_convert_dyn_option_description'] = True
cfg['risotto_globals'] = {}
# initialise variables to store useful informations
# those variables are use during templating
self.templates_dir = {}
self.patches_dir = {}
self.functions_files = {}
self.manuals_dirs = {}
self.tests_dirs = {}
self.modules = {}
functions_files = set()
applicationservices = Applications()
zones_name = {}
rougail = RougailConvert(cfg)
for host_name, datas in self.servers_json['hosts'].items():
for server_name, server_datas in datas['servers'].items():
for zone in server_datas['informations']['zones_name']:
zones_name.setdefault(zone, []).append(server_name)
self.zones = {}
zones_network = ip_network(self.servers_json['zones']['network'])
zone_start_ip = zones_network.network_address
domain_name = self.servers_json['zones']['prefix_domain_name']
for idx, zone_name in enumerate(zones_name):
sub_network = ip_network(f'{zone_start_ip}/28')
if not sub_network.subnet_of(zones_network):
raise Exception('not enough IP available')
if sub_network.num_addresses < len(zones_name[zone_name]):
#FIXME should try to increase network!
raise Exception(f'network too small for zone {zone_name}')
if idx == 0:
zone_domaine_name = domain_name
else:
zone_domaine_name = zone_name + '.' + domain_name
network = sub_network.network_address
self.zones[zone_name] = {'domain_name': zone_domaine_name,
'network': str(sub_network),
'host_ip': str(network + 1),
'start_ip': str(network + 2)
}
zone_start_ip = str(sub_network.broadcast_address + 1)
for host_name, datas in self.servers_json['hosts'].items():
# load modules associate to this host
modules_name = set()
for name, mod_datas in datas['servers'].items():
if not 'applicationservice' in mod_datas:
raise Exception(f'applicationservice is mandatory for "{name}"')
modules_name.add(mod_datas['applicationservice'])
# load modules informations from config files
modules = Modules(applicationservices,
datas['applicationservice_provider'],
modules_name,
datas['applicationservice'],
)
# load host
module_info = modules.get('host')
cfg['risotto_globals'][host_name] = {'global:server_name': host_name,
'global:module_name': 'host',
'global:host_install_dir': abspath(INSTALL_DIR),
}
functions_files |= set(module_info.functions_file)
self.load_dictionaries(cfg,
module_info,
host_name,
rougail,
)
# load servers
modules_info = {}
for server_name, server_datas in datas['servers'].items():
module_info = modules.get(server_datas['applicationservice'])
zones_name = server_datas['informations']['zones_name']
values = [f'{server_name}.{self.zones[zone_name]["domain_name"]}' for zone_name in zones_name]
if server_datas['applicationservice'] == 'tls':
true_host_name = f'{server_name}.{self.zones[list(self.zones)[0]]["domain_name"]}'
else:
true_host_name = values[0]
cfg['risotto_globals'][true_host_name] = {'global:host_name': host_name,
'global:server_name': true_host_name,
'global:server_names': values,
'global:zones_name': zones_name,
'global:zones_list': list(range(len(zones_name))),
'global:module_name': server_datas['applicationservice'],
}
server_datas['server_name'] = true_host_name
functions_files |= set(module_info.functions_file)
self.load_dictionaries(cfg,
module_info,
true_host_name,
rougail,
)
modules_info[module_info.module_name] = module_info.depends
self.modules[host_name] = modules_info
cfg['functions_file'] = list(functions_files)
self.tiram_obj = rougail.save(TIRAMISU_CACHE)
def add_tls(self):
zones = set()
dns_module_name = None
for host in self.servers_json['hosts'].values():
zones = [None, None]
for server_name, datas in host['servers'].items():
if datas['applicationservice'] == 'tls':
raise Exception(f'forbidden module name "tls" for server {server_name}')
#FIXME use provider!
if datas['applicationservice'] == 'nginx-reverse-proxy' and len(datas['informations']['zones_name']) > 0:
if dns_module_name:
break
# always add tls machine in second zone of reverse proxy
zones[1] = datas['informations']['zones_name'][0]
if datas['applicationservice'] == 'unbound':
# always add tls machine in second zone of reverse proxy
zones[0] = datas['informations']['zones_name'][0]
if None in zones:
zones = []
else:
if zones[0] == zones[1]:
zones = [zones[0]]
host['servers']['tls'] = {'applicationservice': 'tls',
'informations': {'zones_name': list(zones)},
}
def load_dictionaries(self, cfg, module_info, server_name, rougail):
if not module_info.dictionaries_dir:
raise Exception(f'server "{server_name}" has any dictionaries!')
cfg['dictionaries_dir'] = module_info.dictionaries_dir
cfg['extra_dictionaries'] = module_info.extra_dictionaries
cfg['functions_file'] = module_info.functions_file
rougail.load_dictionaries(path_prefix=server_name)
self.templates_dir[server_name] = module_info.templates_dir
self.patches_dir[server_name] = module_info.patches_dir
self.functions_files[server_name] = module_info.functions_file
self.manuals_dirs[server_name] = module_info.manuals
self.tests_dirs[server_name] = module_info.tests
async def tiramisu_file_to_tiramisu(self):
# l
tiramisu_space = FUNCTIONS.copy()
try:
exec(self.tiram_obj, None, tiramisu_space)
except Exception as err:
print(self.tiram_obj)
raise Exception(f'unknown error when load tiramisu object {err}') from err
if self.original_display_name:
display_name = None
else:
display_name = tiramisu_display_name
self.config = await Config(tiramisu_space['option_0'],
display_name=display_name,
)
async def load_values_and_informations(self):
config = self.config
await config.property.read_write()
await config.property.pop('validator')
await config.property.pop('cache')
load_zones(self.zones, self.servers_json['hosts'])
await config.information.set('zones', self.zones)
for host_name, hosts_datas in self.servers_json['hosts'].items():
information = config.option(normalize_family(host_name)).information
await information.set('module', 'host')
await information.set('templates_dir', self.templates_dir[host_name])
await information.set('patches_dir', self.patches_dir[host_name])
await information.set('functions_files', self.functions_files[host_name])
await self.set_values(host_name, config, hosts_datas)
for datas in hosts_datas['servers'].values():
server_name = datas['server_name']
information = config.option(normalize_family(server_name)).information
await information.set('module', datas['applicationservice'])
await information.set('templates_dir', self.templates_dir[server_name])
await information.set('patches_dir', self.patches_dir[server_name])
await information.set('functions_files', self.functions_files[server_name])
await information.set('manuals_dirs', self.manuals_dirs[server_name])
await information.set('tests_dirs', self.tests_dirs[server_name])
await self.set_values(server_name, config, datas)
await config.information.set('copy_tests', False)
# FIXME only one host_name is supported
await config.information.set('modules', self.modules[host_name])
# await config.information.set('modules', {module_name: module_info.depends for module_name, module_info in self.module_infos.items() if module_name in modules})
await config.property.add('cache')
if self.valid_mandatories:
messages = await valid_mandatories(config)
if messages:
msg = ''
for title, variables in messages.items():
msg += '\n' + title + '\n'
msg += '\n'.join(variables)
raise Exception(msg)
await config.property.read_only()
with open(VALUES_CACHE, 'w') as fh:
json_dump(await config.value.exportation(), fh)
with open(INFORMATIONS_CACHE, 'w') as fh:
json_dump(await config.information.exportation(), fh)
async def set_values(self,
server_name,
config,
datas,
):
if 'values' not in datas:
return
if not isinstance(datas['values'], dict):
raise Exception(f'Values of "{server_name}" are not a dict: {datas["values"]}')
server_path = normalize_family(server_name)
await config.owner.set(self.config_file)
for vpath, value in datas['values'].items():
path = f'{server_path}.{vpath}'
try:
if isinstance(value, dict):
for idx, val in value.items():
await config.option(path, int(idx)).value.set(val)
else:
await config.option(path).value.set(value)
except Exception as err:
await value_pprint(await config.value.dict(), config)
error_msg = f'cannot configure variable {vpath} for server "{server_name}": {err}'
raise Exception(error_msg) from err
await config.owner.set('user')
class LoaderCache(Loader):
def load_tiramisu_file(self):
with open(TIRAMISU_CACHE) as fh:
self.tiram_obj = fh.read()
async def load_values_and_informations(self):
with open(VALUES_CACHE, 'r') as fh:
await self.config.value.importation(json_load(fh))
with open(INFORMATIONS_CACHE, 'r') as fh:
informations = json_load(fh)
# null is not a valid key in json => 'null'
informations[None] = informations.pop('null')
await self.config.information.importation(informations)
async def load(clean_directories=False,
hide_secret=False,
original_display_name: bool=False,
valid_mandatories: bool=True,
copy_tests: bool=False,
):
if isfile(TIRAMISU_CACHE) and isfile(VALUES_CACHE) and isfile(INFORMATIONS_CACHE):
loader_obj = LoaderCache
else:
loader_obj = Loader
loader = loader_obj(clean_directories,
hide_secret,
original_display_name,
valid_mandatories,
)
loader.load_tiramisu_file()
await loader.tiramisu_file_to_tiramisu()
await loader.load_values_and_informations()
config = loader.config
await config.property.read_only()
await config.information.set('copy_tests', copy_tests)
await config.cache.reset()
return config
async def build_files(hostname: str,
only_machine: str,
just_copy: bool,
copy_tests: bool,
template: str=None,
) -> None:
with open(CONFIG_FILE, 'r') as server_fh:
servers_json = yaml_load(server_fh, Loader=SafeLoader)
config = await load(copy_tests=copy_tests)
machines = [await subconfig.option.description() for subconfig in await config.option.list(type='optiondescription')]
certificates = {'certificates': {},
'configuration': servers_json['certificates'],
}
# get certificates informations
tls_machine = None
for machine in machines:
if machine.startswith('tls.'):
tls_machine = machine
continue
if hostname is None:
# FIXME multi host!
hostname = await config.option(normalize_family(machine)).option('general.host_name').value.get()
if just_copy:
continue
is_host = machine == hostname
if is_host:
continue
machine_config = config.option(normalize_family(machine))
certificate_names = []
private_names = []
for service in await machine_config.option('services').option.list('optiondescription'):
if not await service.option('activate').value.get():
continue
# if await service.option('manage').value.get():
# certificate_type = 'server'
# else:
# certificate_type = 'client'
tls_ca_directory = await machine_config.option('general.tls_ca_directory').value.get()
tls_cert_directory = await machine_config.option('general.tls_cert_directory').value.get()
tls_key_directory = await machine_config.option('general.tls_key_directory').value.get()
try:
for certificate in await service.option('certificates').option.list('all'):
if not await certificate.option('activate').value.get():
continue
certificate_data = await certificate.value.dict()
certificate_data['type'] = await certificate.information.get('type')
certificate_data['authority'] = join(tls_ca_directory, await certificate.information.get('authority') + '.crt')
certificate_data['format'] = await certificate.information.get('format')
is_list_name = isinstance(certificate_data['name'], list)
is_list_domain = isinstance(certificate_data['domain'], list)
if is_list_name != is_list_domain:
raise Exception('certificate name and domain name must be a list together')
if 'provider' not in certificate_data:
certificate_data['provider'] = 'autosigne'
if is_list_name:
if len(certificate_data['name']) != len(certificate_data['domain']):
raise Exception('certificate name and domain name must have same lenght')
for idx, certificate_name in enumerate(certificate_data['name']):
cert_data = certificate_data.copy()
if certificate_data['format'] == 'cert_key':
cert_data['name'] = join(tls_cert_directory, certificate_name + '.crt')
private = join(tls_key_directory, certificate_name + '.key')
if private in private_names:
raise Exception(f'duplicate private key {private} for {machine}')
cert_data['private'] = private
private_names.append(private)
else:
cert_data['name'] = join(tls_key_directory, certificate_name + '.pem')
cert_data['domain'] = certificate_data['domain'][idx]
if cert_data['name'] in certificate_names:
raise Exception(f'duplicate certificate {cert_data["name"]} for {machine}')
certificates['certificates'].setdefault(machine, []).append(cert_data)
certificate_names.append(cert_data['name'])
else:
name = certificate_data['name']
if certificate_data['format'] == 'cert_key':
certificate_data['name'] = join(tls_cert_directory, name + '.crt')
private = join(tls_key_directory, name + '.key')
if private in private_names:
raise Exception(f'duplicate private key {private} for {machine}')
certificate_data['private'] = private
else:
certificate_data['name'] = join(tls_key_directory, name + '.pem')
if certificate_data['name'] in certificate_names:
raise Exception(f'duplicate certificate {certificate_data["name"]} for {machine}')
certificate_names.append(certificate_data['name'])
certificates['certificates'].setdefault(machine, []).append(certificate_data)
except AttributeError:
pass
directories = {}
for machine in machines:
if just_copy and hostname == machine:
continue
if only_machine and only_machine != machine:
continue
await templates(machine,
config,
just_copy=just_copy,
copy_manuals=True,
template=template,
extra_variables=certificates,
)
is_host = machine == hostname
if is_host:
directories[machine] = '/usr/local/lib'
elif not just_copy:
machine_config = config.option(normalize_family(machine))
directories[machine] = await machine_config.option('general.config_dir').value.get()
if only_machine:
return directories
if only_machine:
raise Exception(f'cannot find machine {only_machine}: {machines}')
return directories, certificates