risotto/src/risotto/machine.py

601 lines
28 KiB
Python

from .utils import MULTI_FUNCTIONS, load_zones, value_pprint, RISOTTO_CONFIG, EXTRA_ANNOTATORS, ROUGAIL_NAMESPACE, ROUGAIL_NAMESPACE_DESCRIPTION
from .image import Applications, Modules, valid_mandatories, applicationservice_copy
from rougail import RougailConfig, RougailConvert
from os import remove, makedirs, listdir, chmod
from os.path import isfile, isdir, abspath, join, dirname
from pickle import dump as pickle_dump, load as pickle_load
from yaml import load as yaml_load, SafeLoader
from ipaddress import IPv4Interface, ip_network
#
from tiramisu import Config
from rougail.utils import normalize_family
from rougail import RougailSystemdTemplate
from shutil import copy2, copytree, rmtree
def tiramisu_display_name(kls,
dyn_name: 'Base'=None,
suffix: str=None,
) -> str:
# FIXME
if dyn_name is not None:
name = kls.impl_getpath() + str(suffix)
else:
name = kls.impl_getpath()
return name
CONFIG_FILE = 'servers.yml'
TIRAMISU_CACHE = 'tiramisu_cache.py'
VALUES_CACHE = 'values_cache.pickle'
INFORMATIONS_CACHE = 'informations_cache.pickle'
INSTALL_DIR = RISOTTO_CONFIG['directories']['dest']
INSTALL_CONFIG_DIR = 'configurations'
INSTALL_TMPL_DIR= 'templates'
INSTALL_IMAGES_DIR = 'images_files'
INSTALL_TESTS_DIR = 'tests'
def copy(src_file, dst_file):
if isdir(src_file):
if not isdir(dst_file):
makedirs(dst_file)
for subfilename in listdir(src_file):
if not isfile(dst_file):
src = join(src_file, subfilename)
dst = join(dst_file, subfilename)
if isfile(src):
copy2(src, dst)
else:
copytree(src, dst)
elif not isfile(dst_file):
dst = dirname(dst_file)
if not isdir(dst):
makedirs(dst)
if isfile(src_file):
copy2(src_file, dst_file)
else:
copytree(src_file, dst_file)
def re_create(dir_name):
if isdir(dir_name):
rmtree(dir_name)
makedirs(dir_name)
def remove_cache():
if isfile(TIRAMISU_CACHE):
remove(TIRAMISU_CACHE)
if isfile(VALUES_CACHE):
remove(VALUES_CACHE)
if isfile(INFORMATIONS_CACHE):
remove(INFORMATIONS_CACHE)
def templates(server_name,
config,
just_copy=False,
copy_manuals=False,
template=None,
extra_variables=None,
):
subconfig = config.option(normalize_family(server_name))
try:
subconfig.get()
except:
servers = [server.description() for server in config.list('optiondescription')]
raise Exception(f'cannot find server name "{server_name}": {servers}')
rougailconfig = RougailConfig.copy()
rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE
rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
rougailconfig['tmp_dir'] = 'tmp'
rougailconfig['templates_dir'] = subconfig.information.get('templates_dir')
rougailconfig['patches_dir'] = subconfig.information.get('patches_dir')
rougailconfig['functions_file'] = subconfig.information.get('functions_files')
module = subconfig.information.get('module')
is_host = module == 'host'
if is_host:
rougailconfig['systemd_tmpfile_delete_before_create'] = True
if just_copy:
raise Exception('cannot generate template with option just_copy for a host')
else:
rougailconfig['systemd_tmpfile_delete_before_create'] = False
#rougailconfig['systemd_tmpfile_factory_dir'] = '/usr/local/lib'
if not just_copy:
rougailconfig['destinations_dir'] = join(INSTALL_DIR, INSTALL_CONFIG_DIR, server_name)
else:
rougailconfig['destinations_dir'] = join(INSTALL_DIR, INSTALL_TMPL_DIR, server_name)
re_create(rougailconfig['destinations_dir'])
re_create(rougailconfig['tmp_dir'])
engine = RougailSystemdTemplate(subconfig,
rougailconfig,
)
if just_copy:
# for all engine to none
ori_engines = {}
for eng in engine.engines:
if eng == 'none':
continue
ori_engines[eng] = engine.engines[eng]
engine.engines[eng] = engine.engines['none']
try:
if not template:
engine.instance_files(extra_variables=extra_variables)
else:
engine.instance_file(template, extra_variables=extra_variables)
except Exception as err:
print()
print(f'=== Configuration: {server_name} ===')
try:
values = subconfig.value.dict()
value_pprint(values, subconfig)
except:
pass
raise err from err
if just_copy:
for eng, old_engine in ori_engines.items():
engine.engines[eng] = old_engine
secrets_dir = join(rougailconfig['destinations_dir'], 'secrets')
if isdir(secrets_dir):
chmod(secrets_dir, 0o700)
if copy_manuals and not is_host:
dest_dir = join(INSTALL_DIR, INSTALL_IMAGES_DIR, module)
if not isdir(dest_dir):
for manual in subconfig.information.get('manuals_dirs'):
for filename in listdir(manual):
src_file = join(manual, filename)
dst_file = join(dest_dir, filename)
copy(src_file, dst_file)
copy_tests = config.information.get('copy_tests')
if copy_tests and not is_host:
dest_dir = join(INSTALL_DIR, INSTALL_TESTS_DIR, module)
if not isdir(dest_dir):
for tests in subconfig.information.get('tests_dirs'):
for filename in listdir(tests):
src_file = join(tests, filename)
dst_file = join(dest_dir, filename)
copy(src_file, dst_file)
class Loader:
def __init__(self,
hide_secret,
original_display_name,
valid_mandatories,
config_file=CONFIG_FILE,
):
self.hide_secret = hide_secret
self.original_display_name = original_display_name
self.valid_mandatories = valid_mandatories
self.config_file = config_file
def load_tiramisu_file(self):
"""Load config file (servers.yml) and build tiramisu file with dataset informations
"""
with open(self.config_file, 'r') as server_fh:
self.servers_json = yaml_load(server_fh, Loader=SafeLoader)
self.add_tls()
# set global rougail configuration
cfg = RougailConfig.copy()
cfg['variable_namespace'] = ROUGAIL_NAMESPACE
cfg['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
cfg['multi_functions'] = MULTI_FUNCTIONS
cfg['extra_annotators'] = EXTRA_ANNOTATORS
cfg['force_convert_dyn_option_description'] = True
cfg['risotto_globals'] = {}
# initialise variables to store useful informations
# those variables are use during templating
self.templates_dir = {}
self.patches_dir = {}
self.functions_files = {}
self.manuals_dirs = {}
self.tests_dirs = {}
self.modules = {}
functions_files = set()
applicationservices = Applications()
zones_name = {}
rougail = RougailConvert(cfg)
for host_name, datas in self.servers_json['hosts'].items():
for server_name, server_datas in datas['servers'].items():
if 'provider_zone' not in server_datas and 'zones_name' not in server_datas:
raise Exception(f'cannot find "zones_name" attribute for server "{server_name}"')
if 'provider_zone' in server_datas:
zones_name.setdefault(server_datas['provider_zone'], []).append(server_name)
if 'zones_name' not in server_datas:
server_datas['zones_name'] = []
if server_datas['provider_zone'] in server_datas['zones_name']:
raise Exception(_('provider_zone "{server_datas["provider_zone"]}" must not be in "zones" "{server_datas["zones_name"]}"'))
# external zone is better in first place
if server_datas['zones_name'] and self.servers_json['zones']['external_zone'] == server_datas['zones_name'][0]:
server_datas['zones_name'].append(server_datas['provider_zone'])
else:
server_datas['zones_name'].insert(0, server_datas['provider_zone'])
# if server_datas['zones_name'] and server_datas['provider_zone'] == self.servers_json['zones']['external_zone']:
# server_datas['zones_name'].insert(0, server_datas['provider_zone'])
# else:
# server_datas['zones_name'].append(server_datas['provider_zone'])
for zone in server_datas['zones_name']:
zones_name.setdefault(zone, []).append(server_name)
self.zones = {}
zones_network = ip_network(self.servers_json['zones']['network'])
zone_start_ip = zones_network.network_address
domain_name = self.servers_json['zones']['prefix_domain_name']
for zone_name in zones_name:
len_zone = len(zones_name[zone_name])
for zone_cidr in [29, 28, 27, 26]:
try:
sub_network = ip_network(f'{zone_start_ip}/{zone_cidr}')
except ValueError:
# calc network address for this mask
zone_start_ip = IPv4Interface(f'{zone_start_ip}/{zone_cidr}').network.broadcast_address + 1
sub_network = ip_network(f'{zone_start_ip}/{zone_cidr}')
if not sub_network.subnet_of(zones_network):
raise Exception('not enough IP available')
length = sub_network.num_addresses - 3 # network + broadcast + host
if length >= len_zone:
break
else:
raise Exception(f'network too small for zone "{zone_name}" ({sub_network.num_addresses - 2} < {len_zone})')
if self.servers_json['zones']['external_zone'] == zone_name:
zone_domaine_name = domain_name
else:
zone_domaine_name = zone_name + '.' + domain_name
network = sub_network.network_address
self.zones[zone_name] = {'domain_name': zone_domaine_name,
'network': str(sub_network),
'host_ip': str(network + 1),
'host_name': host_name.split('.', 1)[0],
'length': length,
'start_ip': str(network + 2)
}
zone_start_ip = str(sub_network.broadcast_address + 1)
for host_name, datas in self.servers_json['hosts'].items():
# load modules associate to this host
modules_name = set()
for name, mod_datas in datas['servers'].items():
if not 'applicationservice' in mod_datas:
raise Exception(f'applicationservice is mandatory for "{name}"')
modules_name.add(mod_datas['applicationservice'])
# load modules informations from config files
modules = Modules(applicationservices,
datas['applicationservice_provider'],
modules_name,
datas['applicationservice'],
)
# load host
module_info = modules.get('host')
tls_host_name = f'{server_name}.{self.zones[list(self.zones)[0]]["domain_name"]}'
short_host_name = host_name.split('.', 1)[0]
values = [f'{short_host_name}.{self.zones[zone_name]["domain_name"]}' for zone_name in zones_name]
cfg['risotto_globals'][host_name] = {'global:server_name': host_name,
'global:server_names': values,
'global:zones_name': list(self.zones),
'global:module_name': 'host',
'global:host_install_dir': abspath(INSTALL_DIR),
'global:tls_server': tls_host_name,
}
functions_files |= set(module_info.functions_file)
self.load_dictionaries(cfg,
module_info,
host_name,
rougail,
)
# load servers
modules_info = {}
for server_name, server_datas in datas['servers'].items():
module_info = modules.get(server_datas['applicationservice'])
zones_name = server_datas['zones_name']
values = [f'{server_name}.{self.zones[zone_name]["domain_name"]}' for zone_name in zones_name]
if server_datas['applicationservice'] == 'tls':
true_host_name = f'{server_name}.{self.zones[list(self.zones)[0]]["domain_name"]}'
else:
true_host_name = values[0]
cfg['risotto_globals'][true_host_name] = {'global:host_name': host_name,
'global:server_name': true_host_name,
'global:server_names': values,
'global:zones_name': zones_name,
'global:zones_list': list(range(len(zones_name))),
'global:module_name': server_datas['applicationservice'],
'global:prefix_domain_name': self.servers_json['zones']['prefix_domain_name']
}
if 'provider_zone' in server_datas:
cfg['risotto_globals'][true_host_name]['global:provider_zone'] = server_datas['provider_zone']
server_datas['server_name'] = true_host_name
functions_files |= set(module_info.functions_file)
self.load_dictionaries(cfg,
module_info,
true_host_name,
rougail,
)
modules_info[module_info.module_name] = module_info.depends
self.modules[host_name] = modules_info
cfg['functions_file'] = list(functions_files)
self.tiram_obj = rougail.save(TIRAMISU_CACHE)
with open(TIRAMISU_CACHE, 'a') as cache:
cache.write(f"""#from pickle import load
#config = Config(option_0)
#config.property.read_only()
#with open('{VALUES_CACHE}', 'rb') as fh:
# config.value.importation(load(fh))
#with open('{INFORMATIONS_CACHE}', 'rb') as fh:
# config.information.importation(load(fh))
#print(config.value.mandatory())
""")
def add_tls(self):
dns_module_name = None
for host in self.servers_json['hosts'].values():
zones = [self.servers_json['zones']['external_zone'], None]
for server_name, datas in host['servers'].items():
if not 'applicationservice' in datas:
raise Exception(f'cannot find applicationservice for "{server_name}"')
if datas['applicationservice'] == 'tls':
raise Exception(f'forbidden module name "tls" for server "{server_name}"')
#FIXME use provider!
if datas['applicationservice'] == 'nginx-reverse-proxy' and len(datas['zones_name']) > 0:
if dns_module_name:
break
zones[1] = datas['provider_zone']
if zones[0] == zones[1] or not zones[1]:
zones.pop(1)
host['servers']['tls'] = {'applicationservice': 'tls',
'zones_name': zones,
}
def load_dictionaries(self, cfg, module_info, server_name, rougail):
if not module_info.dictionaries_dir:
raise Exception(f'server "{server_name}" has any dictionaries!')
cfg['dictionaries_dir'] = module_info.dictionaries_dir
cfg['extra_dictionaries'] = module_info.extra_dictionaries
cfg['functions_file'] = module_info.functions_file
rougail.load_dictionaries(path_prefix=server_name)
self.templates_dir[server_name] = module_info.templates_dir
self.patches_dir[server_name] = module_info.patches_dir
self.functions_files[server_name] = module_info.functions_file
self.manuals_dirs[server_name] = module_info.manuals
self.tests_dirs[server_name] = module_info.tests
def tiramisu_file_to_tiramisu(self):
# l
tiramisu_space = {}
try:
exec(self.tiram_obj, None, tiramisu_space)
except Exception as err:
raise Exception(f'unknown error when load tiramisu object: "{err}" see the file "{TIRAMISU_CACHE}" for more details') from err
if self.original_display_name:
display_name = None
else:
display_name = tiramisu_display_name
self.config = Config(tiramisu_space['option_0'],
display_name=display_name,
)
def load_values_and_informations(self):
config = self.config
config.property.read_write()
config.property.remove('validator')
config.property.remove('cache')
load_zones(self.zones, self.servers_json['hosts'])
config.information.set('zones', self.zones)
for host_name, hosts_datas in self.servers_json['hosts'].items():
information = config.option(normalize_family(host_name)).information
information.set('module', 'host')
information.set('templates_dir', self.templates_dir[host_name])
information.set('patches_dir', self.patches_dir[host_name])
information.set('functions_files', self.functions_files[host_name])
self.set_values(host_name, config, hosts_datas)
for datas in hosts_datas['servers'].values():
server_name = datas['server_name']
information = config.option(normalize_family(server_name)).information
information.set('module', datas['applicationservice'])
information.set('templates_dir', self.templates_dir[server_name])
information.set('patches_dir', self.patches_dir[server_name])
information.set('functions_files', self.functions_files[server_name])
information.set('manuals_dirs', self.manuals_dirs[server_name])
information.set('tests_dirs', self.tests_dirs[server_name])
self.set_values(server_name, config, datas)
config.information.set('copy_tests', False)
# FIXME only one host_name is supported
config.information.set('modules', self.modules[host_name])
# config.information.set('modules', {module_name: module_info.depends for module_name, module_info in self.module_infos.items() if module_name in modules})
with open(VALUES_CACHE, 'wb') as fh:
pickle_dump(config.value.exportation(), fh)
with open(INFORMATIONS_CACHE, 'wb') as fh:
pickle_dump(config.information.exportation(), fh)
config.property.add('cache')
if self.valid_mandatories:
messages = valid_mandatories(config)
if messages:
msg = ''
for title, variables in messages.items():
msg += '\n' + title + '\n'
msg += '\n'.join(variables)
raise Exception(msg)
config.property.read_only()
def set_values(self,
server_name,
config,
datas,
):
if 'values' not in datas:
return
if not isinstance(datas['values'], dict):
raise Exception(f'Values of "{server_name}" are not a dict: {datas["values"]}')
server_path = normalize_family(server_name)
config.owner.set(self.config_file)
for vpath, value in datas['values'].items():
path = f'{server_path}.{vpath}'
try:
if isinstance(value, dict):
for idx, val in value.items():
config.option(path, int(idx)).value.set(val)
else:
config.option(path).value.set(value)
except Exception as err:
value_pprint(config.value.dict(), config)
error_msg = f'cannot configure variable {vpath} for server "{server_name}": {err}'
raise Exception(error_msg) from err
config.owner.set('user')
class LoaderCache(Loader):
def load_tiramisu_file(self):
with open(TIRAMISU_CACHE) as fh:
self.tiram_obj = fh.read()
def load_values_and_informations(self):
with open(VALUES_CACHE, 'rb') as fh:
self.config.value.importation(pickle_load(fh))
with open(INFORMATIONS_CACHE, 'rb') as fh:
self.config.information.importation(pickle_load(fh))
def load(hide_secret=False,
original_display_name: bool=False,
valid_mandatories: bool=True,
copy_tests: bool=False,
):
if isfile(TIRAMISU_CACHE) and isfile(VALUES_CACHE) and isfile(INFORMATIONS_CACHE):
loader_obj = LoaderCache
else:
loader_obj = Loader
loader = loader_obj(hide_secret,
original_display_name,
valid_mandatories,
)
loader.load_tiramisu_file()
loader.tiramisu_file_to_tiramisu()
loader.load_values_and_informations()
config = loader.config
config.property.read_only()
config.information.set('copy_tests', copy_tests)
config.cache.reset()
return config
def build_files(hostname: str,
only_machine: str,
just_copy: bool,
copy_tests: bool,
template: str=None,
) -> None:
if isdir(INSTALL_DIR):
rmtree(INSTALL_DIR)
makedirs(INSTALL_DIR)
with open(CONFIG_FILE, 'r') as server_fh:
servers_json = yaml_load(server_fh, Loader=SafeLoader)
config = load(copy_tests=copy_tests)
machines = [subconfig.description() for subconfig in config.option.list(type='optiondescription')]
certificates = {'certificates': {},
'configuration': servers_json['certificates'],
}
# get certificates informations
tls_machine = config.option(f'{normalize_family(hostname)}.general.tls_server').value.get()
for machine in machines:
if machine == tls_machine:
continue
if hostname is None:
# FIXME multi host!
hostname = config.option(normalize_family(machine)).option('general.host_name').value.get()
if just_copy:
continue
is_host = machine == hostname
if is_host:
continue
machine_config = config.option(normalize_family(machine))
certificate_names = []
private_names = []
for service in machine_config.option('services').list('optiondescription'):
if not service.option('activate').value.get():
continue
# if service.option('manage').value.get():
# certificate_type = 'server'
# else:
# certificate_type = 'client'
tls_ca_directory = machine_config.option('general.tls_ca_directory').value.get()
tls_cert_directory = machine_config.option('general.tls_cert_directory').value.get()
tls_key_directory = machine_config.option('general.tls_key_directory').value.get()
try:
for certificate in service.option('certificates').list('all'):
if not certificate.option('activate').value.get():
continue
certificate_data = {key.rsplit('.', 1)[1]: value for key, value in certificate.value.dict().items()}
certificate_data['type'] = certificate.information.get('type')
certificate_data['authority'] = join(tls_ca_directory, certificate.information.get('authority') + '.crt')
certificate_data['format'] = certificate.information.get('format')
is_list_name = isinstance(certificate_data['name'], list)
is_list_domain = isinstance(certificate_data['domain'], list)
if is_list_name != is_list_domain:
raise Exception('certificate name and domain name must be a list together')
if 'provider' not in certificate_data:
certificate_data['provider'] = 'autosigne'
if is_list_name:
if len(certificate_data['name']) != len(certificate_data['domain']):
raise Exception('certificate name and domain name must have same length')
for idx, certificate_name in enumerate(certificate_data['name']):
cert_data = certificate_data.copy()
if certificate_data['format'] == 'cert_key':
cert_data['name'] = join(tls_cert_directory, certificate_name + '.crt')
private = join(tls_key_directory, certificate_name + '.key')
if private in private_names:
raise Exception(f'duplicate private key {private} for {machine}')
cert_data['private'] = private
private_names.append(private)
else:
cert_data['name'] = join(tls_key_directory, certificate_name + '.pem')
cert_data['domain'] = certificate_data['domain'][idx]
if cert_data['name'] in certificate_names:
raise Exception(f'duplicate certificate {cert_data["name"]} for {machine}')
certificates['certificates'].setdefault(machine, []).append(cert_data)
certificate_names.append(cert_data['name'])
else:
name = certificate_data['name']
if certificate_data['format'] == 'cert_key':
certificate_data['name'] = join(tls_cert_directory, name + '.crt')
private = join(tls_key_directory, name + '.key')
if private in private_names:
raise Exception(f'duplicate private key {private} for {machine}')
certificate_data['private'] = private
else:
certificate_data['name'] = join(tls_key_directory, name + '.pem')
if certificate_data['name'] in certificate_names:
raise Exception(f'duplicate certificate {certificate_data["name"]} for {machine}')
certificate_names.append(certificate_data['name'])
certificates['certificates'].setdefault(machine, []).append(certificate_data)
except AttributeError:
pass
directories = {}
for machine in machines:
if just_copy and hostname == machine:
continue
if only_machine and only_machine != machine:
continue
templates(machine,
config,
just_copy=just_copy,
copy_manuals=True,
template=template,
extra_variables=certificates,
)
is_host = machine == hostname
if is_host:
directories[machine] = '/usr/local/lib'
elif not just_copy:
machine_config = config.option(normalize_family(machine))
directories[machine] = machine_config.option('general.config_dir').value.get()
if only_machine:
return directories
if only_machine:
raise Exception(f'cannot find machine {only_machine}: {machines}')
return directories, certificates