forked from stove/risotto
298 lines
12 KiB
Python
298 lines
12 KiB
Python
from shutil import copy2, copytree, rmtree
|
|
from os import listdir, makedirs
|
|
from os.path import join, isdir, isfile
|
|
from yaml import load as yaml_load, SafeLoader
|
|
from json import load as json_load
|
|
|
|
from .utils import CONFIGS, RISOTTO_CONFIG, SERVERS, ZONES_SERVER, value_pprint
|
|
from .machine import load_machine_config
|
|
|
|
|
|
FUNCTIONS = 'funcs.py'
|
|
|
|
|
|
class ModuleCfg():
|
|
def __init__(self):
|
|
self.dictionaries_dir = []
|
|
self.modules = []
|
|
self.functions_file = [FUNCTIONS]
|
|
self.templates_dir = []
|
|
self.extra_dictionaries = {}
|
|
self.servers = []
|
|
|
|
|
|
def list_applications() -> dict:
|
|
"""
|
|
{<applicationservice>: seed/<applicationservice>
|
|
"""
|
|
dataset_directory = RISOTTO_CONFIG['directories']['dataset']
|
|
applications = {}
|
|
for applicationservice in listdir(dataset_directory):
|
|
applicationservice_dir = join(dataset_directory, applicationservice)
|
|
if not isdir(applicationservice_dir):
|
|
continue
|
|
if applicationservice in applications:
|
|
raise Exception(f'multi applicationservice: {applicationservice} ({applicationservice_dir} <=> {applications[applicationservice]})')
|
|
applications[applicationservice] = applicationservice_dir
|
|
return applications
|
|
|
|
|
|
def applicationservice_copy(src_file: str,
|
|
dst_file: str,
|
|
copy_if_not_exists: bool,
|
|
manual_dir: str,
|
|
filename: str,
|
|
) -> None:
|
|
if isdir(src_file):
|
|
if not isdir(dst_file):
|
|
makedirs(dst_file)
|
|
for subfilename in listdir(src_file):
|
|
if not copy_if_not_exists or not isfile(dst_file):
|
|
src = join(src_file, subfilename)
|
|
dst = join(dst_file, subfilename)
|
|
if isfile(src):
|
|
copy2(src, dst)
|
|
else:
|
|
copytree(src, dst)
|
|
elif not copy_if_not_exists or not isfile(dst_file):
|
|
src = join(manual_dir, filename)
|
|
dst = dst_file
|
|
if isfile(src):
|
|
copy2(src, dst)
|
|
else:
|
|
copytree(src, dst)
|
|
|
|
|
|
def load_applicationservice_cfg(appname: str,
|
|
as_dir: str,
|
|
install_dir: str,
|
|
cfg: ModuleCfg,
|
|
copy_manual_dir: bool,
|
|
) -> None:
|
|
cfg.modules.append(appname)
|
|
# dictionaries
|
|
dictionaries_dir = join(as_dir, 'dictionaries')
|
|
if isdir(dictionaries_dir):
|
|
cfg.dictionaries_dir.append(dictionaries_dir)
|
|
# funcs
|
|
funcs_dir = join(as_dir, 'funcs')
|
|
if isdir(funcs_dir):
|
|
for f in listdir(funcs_dir):
|
|
if f.startswith('__'):
|
|
continue
|
|
cfg.functions_file.append(join(funcs_dir, f))
|
|
# templates
|
|
templates_dir = join(as_dir, 'templates')
|
|
if isdir(templates_dir):
|
|
cfg.templates_dir.append(templates_dir)
|
|
# extras
|
|
extras_dir = join(as_dir, 'extras')
|
|
if isdir(extras_dir):
|
|
for extra in listdir(extras_dir):
|
|
extra_dir = join(extras_dir, extra)
|
|
if isdir(extra_dir):
|
|
cfg.extra_dictionaries.setdefault(extra, []).append(extra_dir)
|
|
if copy_manual_dir:
|
|
# manual
|
|
for type in ['image', 'install']:
|
|
manual_dir = join(as_dir, 'manual', type)
|
|
if not isdir(manual_dir):
|
|
continue
|
|
for filename in listdir(manual_dir):
|
|
src_file = join(manual_dir, filename)
|
|
if type == 'image':
|
|
dst_file = join(install_dir, 'manual', filename)
|
|
copy_if_not_exists = False
|
|
else:
|
|
dst_file= join(install_dir, '..', filename)
|
|
copy_if_not_exists = True
|
|
applicationservice_copy(src_file,
|
|
dst_file,
|
|
copy_if_not_exists,
|
|
manual_dir,
|
|
filename,
|
|
)
|
|
|
|
|
|
def load_applicationservice(appname: str,
|
|
added: list,
|
|
install_dir: str,
|
|
cfg: ModuleCfg,
|
|
applications: dict,
|
|
copy_manual_dir: bool,
|
|
providers: dict,
|
|
) -> None:
|
|
if appname not in applications:
|
|
raise Exception(f'cannot find application dependency "{appname}" in application "{module_name}"')
|
|
as_dir = applications[appname]
|
|
applicationservice_file = join(as_dir, 'applicationservice.yml')
|
|
if not isfile(applicationservice_file):
|
|
raise Exception(f'cannot find application service file "{applicationservice_file}"')
|
|
load_applicationservice_cfg(appname,
|
|
as_dir,
|
|
install_dir,
|
|
cfg,
|
|
copy_manual_dir,
|
|
)
|
|
added.append(appname)
|
|
with open(applicationservice_file) as yaml:
|
|
app = yaml_load(yaml, Loader=SafeLoader)
|
|
provider = app.get('provider')
|
|
if provider:
|
|
providers.setdefault(provider, [])
|
|
if appname not in providers[provider]:
|
|
providers[provider].append(appname)
|
|
for xml in app.get('depends', []):
|
|
if xml in added:
|
|
continue
|
|
load_applicationservice(xml,
|
|
added,
|
|
install_dir,
|
|
cfg,
|
|
applications,
|
|
copy_manual_dir,
|
|
providers,
|
|
)
|
|
|
|
|
|
def load_image_informations(install_dir: str,
|
|
datas: dict,
|
|
applications: dict,
|
|
copy_manual_dir: bool,
|
|
providers: dict,
|
|
) -> ModuleCfg:
|
|
cfg = ModuleCfg()
|
|
added = []
|
|
for applicationservice in datas['applicationservices']:
|
|
load_applicationservice(applicationservice,
|
|
added,
|
|
install_dir,
|
|
cfg,
|
|
applications,
|
|
copy_manual_dir,
|
|
providers,
|
|
)
|
|
return cfg
|
|
|
|
|
|
async def load_informations(server_name, datas, config):
|
|
await config.information.set('server_name', server_name)
|
|
if 'informations' not in datas:
|
|
return
|
|
for information, value in datas['informations'].items():
|
|
await config.information.set(information, value)
|
|
if 'extra_domainnames' in datas['informations']:
|
|
for idx, extra_domainname in enumerate(datas['informations']['extra_domainnames']):
|
|
if extra_domainname in CONFIGS:
|
|
raise Exception(f'server "{server_name}" is duplicate')
|
|
value = list(CONFIGS[server_name])
|
|
value[4] = idx + 1
|
|
CONFIGS[extra_domainname] = tuple(value)
|
|
|
|
|
|
async def set_values(server_name, config, datas):
|
|
try:
|
|
if 'values' in datas:
|
|
for path, value in datas['values'].items():
|
|
if isinstance(value, dict):
|
|
for idx, val in value.items():
|
|
await config.option(path, int(idx)).value.set(val)
|
|
else:
|
|
await config.option(path).value.set(value)
|
|
except Exception as err:
|
|
await value_pprint(await config.value.dict(), config)
|
|
error_msg = f'cannot configure server "{server_name}": {err}'
|
|
raise Exception(error_msg) from err
|
|
#await config.value.dict()
|
|
|
|
|
|
async def valid_mandatories(server_name, config):
|
|
mandatories = await config.value.mandatory()
|
|
if mandatories:
|
|
print()
|
|
print(f'=== Configuration: {server_name} ===')
|
|
await config.property.pop('mandatory')
|
|
await value_pprint(await config.value.dict(), config)
|
|
raise Exception(f'server "{server_name}" has mandatories variables without values "{", ".join(mandatories)}"')
|
|
|
|
|
|
def load_config(copy_manual_dir=False, clean_directories=False):
|
|
module_infos = {}
|
|
applications = list_applications()
|
|
with open('servers.json', 'r') as server_fh:
|
|
jsonfile = json_load(server_fh)
|
|
SERVERS.update(jsonfile['servers'])
|
|
modules = jsonfile['modules']
|
|
for module_name, datas in modules.items():
|
|
providers = {}
|
|
install_dir = join(RISOTTO_CONFIG['directories']['dest'], module_name)
|
|
if clean_directories:
|
|
if isdir(install_dir):
|
|
rmtree(install_dir)
|
|
makedirs(install_dir)
|
|
module_infos[module_name] = {'infos': load_image_informations(install_dir,
|
|
datas,
|
|
applications,
|
|
copy_manual_dir,
|
|
providers,
|
|
),
|
|
'providers': providers,
|
|
'install_dir': install_dir,
|
|
}
|
|
return module_infos
|
|
|
|
|
|
|
|
async def load(display_name=None,
|
|
clean_directories=False,
|
|
copy_manual_dir=False,
|
|
hide_secret=False,
|
|
):
|
|
# load images
|
|
module_infos = load_config(copy_manual_dir, clean_directories)
|
|
# load machines
|
|
ZONES_SERVER['providers'] = {}
|
|
for server_name, datas in SERVERS.items():
|
|
if server_name in CONFIGS:
|
|
raise Exception(f'server "{server_name}" is duplicate')
|
|
module_info = module_infos[datas['module']]
|
|
CONFIGS[server_name] = await load_machine_config(server_name,
|
|
datas,
|
|
module_info,
|
|
display_name=display_name,
|
|
)
|
|
if module_info['providers'] and 'informations' in datas and 'zones_name' in datas['informations']:
|
|
for zone_idx, zone_name in enumerate(datas['informations']['zones_name']):
|
|
if not zone_idx:
|
|
sname = server_name
|
|
else:
|
|
sname = datas['informations']['extra_domainnames'][zone_idx - 1]
|
|
ZONES_SERVER['providers'].setdefault(zone_name, {})
|
|
for provider in module_info['providers']:
|
|
ZONES_SERVER['providers'][zone_name].setdefault(provider, []).append(sname)
|
|
# set servers.json values
|
|
for server_name, datas in SERVERS.items():
|
|
config = CONFIGS[server_name]['config']
|
|
await load_informations(server_name, datas, config)
|
|
await config.property.read_write()
|
|
if hide_secret:
|
|
try:
|
|
await config.option('general.hide_secret').value.set(True)
|
|
except AttributeError:
|
|
pass
|
|
await set_values(server_name, config, datas)
|
|
await config.property.read_only()
|
|
# force calculates all values (for linked values)
|
|
for server_name in SERVERS:
|
|
config = CONFIGS[server_name]['config']
|
|
await config.property.pop('mandatory')
|
|
try:
|
|
await config.value.dict()
|
|
except Exception as err:
|
|
raise Exception(f'cannot display config for "{server_name}": {err}')
|
|
await config.property.add('mandatory')
|
|
# validate mandatories values
|
|
for server_name in SERVERS:
|
|
await valid_mandatories(server_name, CONFIGS[server_name]['config'])
|
|
|
|
return module_infos
|