gnunux/risotto
1
0
Fork 0
forked from stove/risotto
Code Issues Pull requests Projects Releases Wiki Activity

reorganise

Browse source
This commit is contained in:
egarette@silique.fr 2022-12-21 16:35:58 +01:00
parent 6e38f1c4d1
commit 212699f571
25 changed files with 1224 additions and 739 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
ansible/__init__.py Normal file
View file

0
ansible/action_plugins/__init__.py Normal file
View file

66
ansible/action_plugins/build_images.py
View file

@ -13,36 +13,46 @@ class ActionModule(ActionBase):
super(ActionModule, self).run(tmp, task_vars)
module_args = self._task.args.copy()
modules = module_args['modules']
dataset_directory = RISOTTO_CONFIG['directories']['dataset']
dataset_directories = RISOTTO_CONFIG['directories']['datasets']
install_dir = join('/tmp/risotto/images')
if isdir(install_dir):
rmtree(install_dir)
for module_name, depends in modules.items():
for depend in depends:
manual = join(dataset_directory, depend, 'manual', 'image')
if not isdir(manual):
continue
for filename in listdir(manual):
src_file = join(manual, filename)
dst_file = join(install_dir, module_name, filename)
if isdir(src_file):
if not isdir(dst_file):
makedirs(dst_file)
for subfilename in listdir(src_file):
if not isfile(dst_file):
src = join(src_file, subfilename)
dst = join(dst_file, subfilename)
if isfile(src):
copy2(src, dst)
else:
copytree(src, dst)
elif not isfile(dst_file):
dst = dirname(dst_file)
if not isdir(dst):
makedirs(dst)
if isfile(src_file):
copy2(src_file, dst_file)
else:
copytree(src_file, dst_file)
for dataset_directory in dataset_directories:
for depend in depends:
manual = join(dataset_directory, depend, 'manual', 'image')
if not isdir(manual):
continue
for filename in listdir(manual):
src_file = join(manual, filename)
dst_file = join(install_dir, module_name, filename)
if isdir(src_file):
if not isdir(dst_file):
makedirs(dst_file)
for subfilename in listdir(src_file):
if not isfile(dst_file):
src = join(src_file, subfilename)
dst = join(dst_file, subfilename)
if isfile(src):
copy2(src, dst)
else:
copytree(src, dst)
elif not isfile(dst_file):
dst = dirname(dst_file)
if not isdir(dst):
makedirs(dst)
if isfile(src_file):
copy2(src_file, dst_file)
else:
copytree(src_file, dst_file)
return dict(ansible_facts=dict({}))
#A REFAIRE ICI tests_dir = join(as_dir, 'tests')
#A REFAIRE ICI if isdir(tests_dir):
#A REFAIRE ICI cfg.tests.append(tests_dir)
#A REFAIRE ICI# for filename in listdir(tests_dir):
#A REFAIRE ICI# src_file = join(tests_dir, filename)
#A REFAIRE ICI# dst_file = join(INSTALL_DIR, 'tests', filename)
#A REFAIRE ICI# applicationservice_copy(src_file,
#A REFAIRE ICI# dst_file,
#A REFAIRE ICI# False,
#A REFAIRE ICI# )

97
ansible/action_plugins/rougail.py
View file

@ -1,46 +1,29 @@
#!/usr/bin/python3
from ansible.plugins.action import ActionBase
from asyncio import run
from shutil import rmtree
from os.path import isdir, join
from os import makedirs
from risotto.machine import templates, load, ROUGAIL_NAMESPACE
from risotto.utils import RISOTTO_CONFIG
from rougail.utils import normalize_family
from os import readlink
from os.path import join, islink
from risotto.machine import load, templates
try:
from ansible.plugins.action import ActionBase
from ansible.module_utils.basic import AnsibleModule
class FakeModule(AnsibleModule):
def __init__(self):
pass
except:
import traceback
traceback.print_exc()
class ActionBase():
def __init__(self, *args, **kwargs):
raise Exception('works only with ansible')
TIRAMISU_CACHE = 'tiramisu_cache.py'
VALUES_CACHE = 'values_cache.py'
INSTALL_DIR = RISOTTO_CONFIG['directories']['dest']
async def build_files(server_name, is_host):
module_infos, rougailconfig, config = await load(TIRAMISU_CACHE,
VALUES_CACHE,
)
subconfig = config.option(normalize_family(server_name))
module_name = await subconfig.option(await subconfig.information.get('provider:global:module_name')).value.get()
module_info = module_infos[module_name]
rougailconfig['tmp_dir'] = 'tmp'
rougailconfig['destinations_dir'] = INSTALL_DIR
rougailconfig['templates_dir'] = module_info['infos'].templates_dir
if is_host:
tmpfile = await subconfig.option(f'{ROUGAIL_NAMESPACE}.host_install_dir').value.get()
rougailconfig['tmpfile_dest_dir'] = f'{tmpfile}'
rougailconfig['default_systemd_directory'] = '/usr/local/lib/systemd'
else:
rougailconfig['tmpfile_dest_dir'] = '/usr/local/lib'
rougailconfig['default_systemd_directory'] = '/systemd'
if isdir(rougailconfig['destinations_dir']):
rmtree(rougailconfig['destinations_dir'])
if isdir(rougailconfig['tmp_dir']):
rmtree(rougailconfig['tmp_dir'])
makedirs(rougailconfig['tmp_dir'])
makedirs(rougailconfig['destinations_dir'])
async def build_files(server_name: str,
just_copy: bool,
) -> None:
config = await load()
await templates(server_name,
subconfig,
rougailconfig,
config,
just_copy=just_copy,
)
@ -48,8 +31,38 @@ class ActionModule(ActionBase):
def run(self, tmp=None, task_vars=None):
super(ActionModule, self).run(tmp, task_vars)
module_args = self._task.args.copy()
name = module_args['hostname']
is_host = module_args['is_host']
root_local = module_args.pop('root_local')
root_remote= module_args.pop('root_remote')
name = module_args.pop('hostname')
is_host = module_args.pop('is_host')
just_copy = module_args.get('just_copy', False)
module_args['root'] = root_remote
run(build_files(name, is_host))
return dict(ansible_facts=dict({}))
run(build_files(name, just_copy))
#
remote = self._execute_module(module_name='compare', module_args=module_args, task_vars=task_vars)
if remote.get('failed'):
raise Exception(f'error in remote action: {remote["module_stdout"]}')
#
module = FakeModule()
modified_files = []
changed = False
for path in module_args['paths']:
full_path = join(root_local, path['name'][1:])
if remote['compare'].get(path['name']):
if remote['compare'][path['name']]['type'] == 'file':
if remote['compare'][path['name']]['shasum'] == module.digest_from_file(full_path, 'sha256'):
continue
else:
# it's a symlink
if islink(full_path) and remote['compare'][path['name']]['name'] == readlink(full_path):
continue
changed = True
modified_files.append(path['name'])
if not is_host:
for old_file in remote['old_files']:
changed = True
# module_args['path'] = old_file
# module_args['state'] = 'absent'
# self._execute_module(module_name='ansible.builtin.file', module_args=module_args, task_vars=task_vars)
return dict(ansible_facts=dict({}), changed=changed)

35
ansible/filter_plugins/fileslist.py
View file

@ -31,22 +31,30 @@ def fileslist(data, is_host=False, name_only=False, prefix=None):
)
for service, service_data in data.items():
if not service_data['activate']:
if service_data['engine'] == 'none' and service_data['type'] == 'service' and not 'overrides' in service_data:
_add(files,
{'owner': 'root', 'group': 'root', 'mode': '0755'},
base_systemd + '/systemd/system/' + service_data['doc'],
name_only,
prefix,
)
if service_data['manage']:
if not service_data.get('undisable', False) and not service_data['engine'] and not service_data.get('target'):
_add(files,
{'owner': 'root', 'group': 'root', 'mode': '0755'},
base_systemd + '/systemd/system/' + service_data['doc'],
name_only,
prefix,
)
else:
if service_data['activate'] and service_data['engine'] != 'none':
if service_data['manage'] and service_data['engine']:
_add(files,
{'owner': 'root', 'group': 'root', 'mode': '0755'},
base_systemd + '/systemd/system/' + service_data['doc'],
name_only,
prefix,
)
if service_data['activate'] and 'overrides' in service_data:
if service_data.get('target'):
_add(files,
{'owner': 'root', 'group': 'root', 'mode': '0755'},
f'/systemd/system/{service_data["target"]}.target.wants/{service_data["doc"]}',
name_only,
prefix,
)
if 'overrides' in service_data:
for override_data in service_data['overrides'].values():
_add(files,
{'owner': 'root', 'group': 'root', 'mode': '0755'},
@ -54,6 +62,13 @@ def fileslist(data, is_host=False, name_only=False, prefix=None):
name_only,
prefix,
)
if 'ip' in service_data:
_add(files,
{'owner': 'root', 'group': 'root', 'mode': '0755'},
base_systemd + '/systemd/system/' + service_data['doc'] + '.d/rougail_ip.conf',
name_only,
prefix,
)
if 'files' not in service_data:
continue
for file_data in service_data['files'].values():
@ -88,6 +103,8 @@ def directorieslist(data):
def machineslist(data, only=None, only_name=False):
srv = []
if only is not None:
if only not in data:
raise Exception(f"cannot find {only} but only {data.keys()}")
if only_name:
srv.append(only)
else:

54
ansible/host.yml
View file

@ -2,6 +2,36 @@
- name: "Populate service facts"
service_facts:
- name: "Packages installation"
apt:
pkg: "{{ vars[inventory_hostname]['general']['host_packages'] }}"
update_cache: yes
state: latest
- name: "Build host files"
rougail:
paths: "{{ vars[inventory_hostname]['services'] | fileslist(is_host=True) }}"
root_local: "{{ host_install_dir }}"
root_remote: "/"
hostname: "{{ inventory_hostname }}"
is_host: True
- name: "Create /usr/local/lib/systemd/system"
file:
path: /usr/local/lib/systemd/system
state: directory
mode: 0755
- name: "Copy service file only if not exists"
when: item.value['manage'] and item.value['activate'] and item.value['doc'].endswith('.service') and not item.value['doc'].endswith('@.service') and item.value['engine'] and item.value['engine'] != 'none'
copy:
src: '{{ host_install_dir }}/usr/local/lib/systemd/system/{{ item.value["doc"] }}'
force: no
dest: '/usr/local/lib/systemd/system/{{ item.value["doc"] }}'
loop: "{{ vars[inventory_hostname]['services'] | dict2items }}"
loop_control:
label: "{{ item.value['doc'] }}"
- name: "Stop services"
when: item.value['manage'] and item.value['activate'] and item.value['doc'].endswith('.service') and not item.value['doc'].endswith('@.service') and item.value['engine'] != 'none'
ansible.builtin.service:
@ -11,18 +41,6 @@
loop_control:
label: "{{ item.value['doc'] }}"
- name: "Packages installation"
apt:
pkg: "{{ vars[inventory_hostname]['general']['host_packages'] }}"
update_cache: yes
state: latest
- name: "Build host files"
local_action:
module: rougail
hostname: "{{ inventory_hostname }}"
is_host: True
- name: "Create host directories"
file: path={{ item }} state=directory mode=0755
loop: "{{ vars[inventory_hostname]['services'] | directorieslist }}"
@ -30,7 +48,7 @@
- name: "Copy systemd-tmpfiles"
when: item.name.startswith('/usr/local/lib/risotto-tmpfiles.d')
ansible.builtin.copy:
src: installations/{{ item.name }}
src: "{{ host_install_dir }}/{{ item.name }}"
dest: "{{ item.name }}"
owner: "{{ item.owner }}"
group: "{{ item.group }}"
@ -49,7 +67,7 @@
- name: "Copy host files"
when: not item.name.startswith('/usr/local/lib/tmpfiles.d')
ansible.builtin.copy:
src: installations/{{ item.name }}
src: "{{ host_install_dir }}/{{ item.name }}"
dest: "{{ item.name }}"
owner: "{{ item.owner }}"
group: "{{ item.group }}"
@ -105,7 +123,7 @@
owner: "root"
group: "root"
mode: "0755"
loop: "{{ lookup('fileglob', '../sbin/*', wantlist=True) | list }}"
loop: "{{ lookup('fileglob', 'sbin/*', wantlist=True) | list }}"
# Images informations
- name: "Remove images tar"
@ -141,3 +159,9 @@
unarchive:
src: "/tmp/risotto/images.tar"
dest: "/var/lib/risotto/images_files"
- name: "Create versions directory"
file:
path: /var/lib/risotto/machines_versions
state: directory
mode: "0700"

90
ansible/inventory.py
View file

@ -5,21 +5,21 @@ Example custom dynamic inventory script for Ansible, in Python.
'''
from argparse import ArgumentParser
from json import dumps, JSONEncoder
from json import load as json_load, dumps, JSONEncoder
from os import remove
from os.path import isfile
from asyncio import run
from traceback import print_exc
from risotto.machine import load
from risotto.image import load_config
from risotto.utils import SERVERS
from risotto.machine import load, TIRAMISU_CACHE, VALUES_CACHE, INFORMATIONS_CACHE, ROUGAIL_NAMESPACE, ROUGAIL_NAMESPACE_DESCRIPTION
from tiramisu import Config
from tiramisu.error import PropertiesOptionError
from rougail.utils import normalize_family
from rougail import RougailSystemdTemplate
from rougail import RougailSystemdTemplate, RougailConfig
from rougail.template.base import RougailLeader, RougailExtra
TIRAMISU_CACHE = 'tiramisu_cache.py'
VALUES_CACHE = 'values_cache.py'
DEBUG = False
class RougailEncoder(JSONEncoder):
@ -38,36 +38,43 @@ class RisottoInventory(object):
parser = ArgumentParser()
parser.add_argument('--list', action='store_true')
parser.add_argument('--host', action='store')
parser.add_argument('--nocache', action='store_true')
parser.add_argument('--debug', action='store_true')
self.args = parser.parse_args()
if self.args.debug:
global DEBUG
DEBUG = True
async def run(self):
if self.args.list:
if self.args.list and self.args.host:
raise Exception('cannot have --list and --host together')
if self.args.list or self.args.nocache:
if isfile(TIRAMISU_CACHE):
remove(TIRAMISU_CACHE)
if isfile(VALUES_CACHE):
remove(VALUES_CACHE)
return await self.do_inventory()
if isfile(INFORMATIONS_CACHE):
remove(INFORMATIONS_CACHE)
config = await load(TIRAMISU_CACHE,
VALUES_CACHE,
INFORMATIONS_CACHE,
)
if self.args.list:
return await self.do_inventory(config)
elif self.args.host:
return await self.get_vars(self.args.host)
return await self.get_vars(config, self.args.host)
raise Exception('pfff')
async def do_inventory(self):
module_infos = load_config(True,
True,
True,
)
servers = []
for server_name, server in SERVERS.items():
module_name = server['module']
if module_name != 'host':
continue
servers.append(server_name)
async def do_inventory(self,
config: Config,
) -> dict:
servers = [await subconfig.option.doc() for subconfig in await config.option.list('optiondescription') if await subconfig.information.get('module') == 'host']
return dumps({
'group': {
'hosts': servers,
'vars': {
# FIXME
'ansible_ssh_host': '192.168.56.156',
# 'ansible_ssh_host': '192.168.0.100',
'ansible_ssh_user': 'root',
'ansible_python_interpreter': '/usr/bin/python3'
}
@ -75,41 +82,42 @@ class RisottoInventory(object):
})
async def get_vars(self,
config: Config,
host_name: str,
) -> dict:
try:
module_infos, rougailconfig, config = await load(TIRAMISU_CACHE,
VALUES_CACHE,
)
except Exception as err:
# import traceback
# traceback.print_exc()
print(err)
exit(1)
ret = {}
modules = set()
for server_name, server in SERVERS.items():
if server['module'] == 'host' and server_name != host_name:
rougailconfig = RougailConfig.copy()
rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE
rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
for subconfig in await config.option.list('optiondescription'):
server_name = await subconfig.option.description()
module_name = await subconfig.option(await subconfig.information.get('provider:global:module_name')).value.get()
if module_name == 'host' and server_name != host_name:
continue
modules.add(server['module'])
subconfig = config.option(normalize_family(server_name))
engine = RougailSystemdTemplate(subconfig, rougailconfig)
await engine.load_variables()
if server['module'] != 'host' and engine.rougail_variables_dict['general']['host'] != host_name:
if module_name != 'host' and engine.rougail_variables_dict['general']['host'] != host_name:
continue
ret[server_name] = engine.rougail_variables_dict
ret['modules'] = {module_name: module_info['infos'].depends for module_name, module_info in module_infos.items() if module_name in modules}
ret['modules'] = await config.information.get('modules')
ret['delete_old_image'] = False
ret['configure_host'] = True
ret['only_machine'] = None
ret['copy_template'] = False
ret['host_install_dir'] = ret[host_name].pop('host_install_dir')
return dumps(ret, cls=RougailEncoder)
# Get the inventory.
async def main():
inv = RisottoInventory()
values = await inv.run()
print(values)
try:
inv = RisottoInventory()
values = await inv.run()
print(values)
except Exception as err:
if DEBUG:
print_exc()
exit(err)
run(main())

68
ansible/library/compare.py Normal file
View file

@ -0,0 +1,68 @@
#!/usr/bin/python3
from time import sleep
from os import fdopen, walk, readlink
from os.path import join, islink
from dbus import SystemBus, Array
from dbus.exceptions import DBusException
from ansible.module_utils.basic import AnsibleModule
def run_module():
# define available arguments/parameters a user can pass to the module
module_args = dict(
root=dict(type='str', required=True),
paths=dict(type='list', required=True),
)
# seed the result dict in the object
# we primarily care about changed and state
# changed is if this module effectively modified the target
# state will include any data that you want your module to pass back
# for consumption, for example, in a subsequent task
result = dict(
changed=False,
compare={},
symlink={},
old_files=[],
)
# the AnsibleModule object will be our abstraction working with Ansible
# this includes instantiation, a couple of common attr would be the
# args/params passed to the execution, as well as if the module
# supports check mode
module = AnsibleModule(
argument_spec=module_args,
supports_check_mode=True
)
root = module.params['root']
if root != '/':
paths = {join(root, path['name'][1:]): path['name'] for path in module.params['paths']}
search_paths = [join(directory, f) for directory, subdirectories, files in walk(root) for f in files]
else:
paths = {path['name']: path['name'] for path in module.params['paths']}
search_paths = paths
for path in search_paths:
if path in paths:
if not islink(path):
result['compare'][paths[path]] = {'type': 'file',
'shasum': module.digest_from_file(path, 'sha256'),
}
else:
result['compare'][paths[path]] = {'type': 'symlink',
'name': readlink(path),
}
else:
result['old_files'].append(path)
module.exit_json(**result)
def main():
run_module()
if __name__ == '__main__':
main()

4
ansible/library/machinectl.py
View file

@ -82,6 +82,7 @@ def start(bus, machines):
error = False
while True:
try:
ret = []
res = remote_object.OpenMachineShell(host,
'',
cmd[0],
@ -91,7 +92,6 @@ def start(bus, machines):
)
fd = res[0].take()
fh = fdopen(fd)
ret = []
while True:
try:
ret.append(fh.readline().strip())
@ -114,7 +114,7 @@ def start(bus, machines):
break
if error:
continue
if ret[0] == 'running':
if ret and ret[0] == 'running':
continue
cmd = ['/usr/bin/systemctl', '--state=failed', '--no-legend', '--no-page']
res = remote_object.OpenMachineShell(host,

113
ansible/machine.yml
View file

@ -6,65 +6,69 @@
file: path=/var/lib/risotto/journals/{{ item.name }} state=directory mode=0755
- name: "Build machine files for {{ item.name }}"
local_action:
module: rougail
rougail:
paths: "{{ vars[item.name]['services'] | fileslist }}"
root_local: "{{ host_install_dir }}"
root_remote: "/var/lib/risotto/configurations/{{ item.name }}"
hostname: "{{ item.name}}"
is_host: False
- name: "Get local informations for {{ item.name }} configuration's file"
local_action:
module: stat
path: "installations{{ file.name }}"
checksum: sha256
get_checksum: yes
follow: true
loop: "{{ vars[item.name]['services'] | fileslist }}"
loop_control:
loop_var: file
label: "{{ file.name }}"
register: local_configuration
- name: "Get remote informations for {{ item.name }} configuration's file"
stat:
path: "/var/lib/risotto/configurations/{{ item.name }}{{ file.name }}"
checksum: sha256
get_checksum: yes
follow: true
loop: "{{ vars[item.name]['services'] | fileslist }}"
loop_control:
loop_var: file
label: "{{ file.name }}"
register: remote_configuration
- name: "Configuration's file is up to date in {{ item.name }}"
debug:
msg: "file is {{ 'out of date' if not file[1].stat.exists or (not 'checksum' in file[0].stat and 'checksum' in file[1].stat) or ('checksum' in file[0].stat and not 'checksum' in file[1].stat) or ('checksum' in file[0].stat and 'checksum' in file[1].stat and file[0].stat.checksum != file[1].stat.checksum) else 'up to date' }}"
changed_when: not file[1].stat.exists or (not 'checksum' in file[0].stat and 'checksum' in file[1].stat) or ('checksum' in file[0].stat and not 'checksum' in file[1].stat) or ('checksum' in file[0].stat and 'checksum' in file[1].stat and file[0].stat.checksum != file[1].stat.checksum)
loop: "{{ local_configuration.results | zip(remote_configuration.results) | list }}"
loop_control:
loop_var: file
label: "{{ file[0]['file']['name'] }}"
ignore_errors: true
register: up_to_date_configuration
- name: "Remove Compressed files for {{ item.name }}"
- name: "Change secrets right"
local_action:
module: file
path: /tmp/new_configurations/{{ item.name }}
state: absent
when: up_to_date_configuration.changed
path: "{{ host_install_dir }}/secrets"
state: directory
mode: 0700
- name: "Compress files for {{ item.name }}"
local_action:
module: archive
path: "installations/"
path: "{{ host_install_dir }}/"
dest: /tmp/new_configurations/{{ item.name }}
format: tar
when: up_to_date_configuration.changed
- name: "Build machine templates for {{ item.name }}"
rougail:
paths: "{{ vars[item.name]['services'] | fileslist }}"
root_local: "{{ host_install_dir }}"
root_remote: "/var/lib/risotto/configurations/{{ item.name }}"
hostname: "{{ item.name}}"
just_copy: true
is_host: False
when: copy_template
register: up_to_date_configuration
- name: "Compress templates for {{ item.name }}"
local_action:
module: archive
path: "../templates/"
dest: /tmp/new_templates/{{ item.name }}
format: tar
when: copy_template
- name: "Remove templates directory for {{ item.name }}"
file:
path: "/var/lib/risotto/templates/{{ item.name }}"
state: absent
when: copy_template
- name: "Create templates directory for {{ item.name }}"
file:
path: "/var/lib/risotto/templates/{{ item.name }}"
state: directory
when: copy_template
- name: "Copy templates for {{ item.name }}"
unarchive:
src: "/tmp/new_templates/{{ item.name }}"
dest: "/var/lib/risotto/templates/{{ item.name }}/"
when: copy_template
- name: "Remove old image {{ vars | modulename(item.name) }}"
file:
path: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}.tar"
path: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}"
state: absent
when: delete_old_image == true
@ -88,27 +92,30 @@
- name: "Check image for {{ item.name }}"
stat:
path: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}.tar"
path: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}"
follow: true
register: register_name
when: system_directory_created.changed
#- name: Print return information from the previous task
# ansible.builtin.debug:
# var: register_name
- name: "Build image for {{ item.name }}"
ansible.builtin.shell: "/usr/local/sbin/build_image {{ vars | modulename(item.name) }}"
when: system_directory_created.changed and not register_name.stat.exists
when: "'stat' in register_name and not register_name.stat.exists"
register: ret
failed_when: ret.rc != 0
- name: "Uncompress machine image for {{ item.name }}"
unarchive:
src: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}.tar"
remote_src: true
dest: /var/lib/machines/{{ item.name }}/
- name: "Copy machine image for {{ item.name }}"
ansible.builtin.shell: "/usr/bin/cp -a --reflink=auto /var/lib/risotto/images/{{ vars | modulename(item.name) }}/* /var/lib/machines/{{ item.name }}"
when: system_directory_created.changed
- name: "SHA machine image for {{ item.name }}"
- name: "Copy machine image version for {{ item.name }}"
ansible.builtin.copy:
src: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}.tar.sha"
src: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}.version"
remote_src: true
dest: "/var/lib/risotto/configurations/sha/{{ item.name }}.sha"
dest: "/var/lib/risotto/machines_versions/{{ item.name }}.version"
owner: "root"
group: "root"
when: system_directory_created.changed

8
ansible/machines.yml
View file

@ -19,6 +19,8 @@
unarchive:
src: "{{ item }}"
dest: /var/lib/risotto/configurations/{{ item | basename }}/
owner: root
group: root
loop: "{{ lookup('fileglob', '/tmp/new_configurations/*', wantlist=True) }}"
- name: "Enable machines"
@ -36,3 +38,9 @@
module: file
path: /tmp/new_configurations
state: absent
- name: "Remove compressed templates directory"
local_action:
module: file
path: /tmp/new_templates
state: absent

18
ansible/playbook.yml
View file

@ -1,7 +1,6 @@
---
#FIXME : si on redemarre a appel tmpfiles.d ....
- name: Risotto
hosts: cloud.silique.fr
hosts: all
tasks:
- name: "Configure the host"
include_tasks: host.yml
@ -20,6 +19,21 @@
state: directory
mode: 0700
- name: "Remove compressed templates files directory"
local_action:
module: file
path: /tmp/new_templates
state: absent
when: copy_template
- name: "Create compressed templates files directory"
local_action:
module: file
path: /tmp/new_templates
state: directory
mode: 0700
when: copy_template
- name: "Prepare machine configuration"
include_tasks: machine.yml
loop: "{{ vars | machineslist(only=only_machine) }}"

100
ansible/sbin/build_image
View file

@ -12,15 +12,13 @@ RISOTTO_DIR="/var/lib/risotto"
RISOTTO_IMAGE_DIR="$RISOTTO_DIR/images"
# image configuration
IMAGE_BASE_RISOTTO_BASE_DIR="$RISOTTO_IMAGE_DIR/image_bases"
IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP="$RISOTTO_IMAGE_DIR/tmp/$IMAGE_NAME"
IMAGE_NAME_RISOTTO_IMAGE_DIR="$RISOTTO_IMAGE_DIR/$IMAGE_NAME"
IMAGE_NAME_RISOTTO_IMAGE_NAME="$RISOTTO_IMAGE_DIR/$IMAGE_NAME".tar
IMAGE_DIR_RECIPIENT_IMAGE="/var/lib/risotto/images_files/$IMAGE_NAME"
#FIXME ou ?
rm -rf "$IMAGE_NAME_RISOTTO_IMAGE_DIR" "$RISOTTO_IMAGE_DIR/tmp"
mkdir -p "$RISOTTO_IMAGE_DIR"
rm -f /var/log/risotto/build_image.log
mkdir -p "$RISOTTO_IMAGE_DIR" "$RISOTTO_IMAGE_DIR/tmp/" /var/log/risotto
PKG=""
BASE_DIR=""
for script in $(ls "$IMAGE_DIR_RECIPIENT_IMAGE"/preinstall/*.sh 2> /dev/null); do
@ -29,93 +27,89 @@ done
if [ -z "$OS_NAME" ]; then
echo "NO OS NAME DEFINED"
exit 0
exit 1
fi
if [ -z "$RELEASEVER" ]; then
echo "NO RELEASEVER DEFINED"
exit 0
exit 1
fi
if [ -z "$INSTALL_TOOL" ]; then
echo "NO INSTALL TOOL DEFINED"
exit 0
exit 1
fi
BASE_NAME="$OS_NAME-$RELEASEVER"
BASE_DIR="$IMAGE_BASE_RISOTTO_BASE_DIR/$BASE_NAME"
BASE_TAR="$IMAGE_BASE_RISOTTO_BASE_DIR-$BASE_NAME".tar
TMP_BASE_DIR="$IMAGE_BASE_RISOTTO_BASE_DIR/tmp/$BASE_NAME"
BASE_PKGS_FILE="$IMAGE_BASE_RISOTTO_BASE_DIR-$BASE_NAME.pkgs"
BASE_LOCK="$IMAGE_BASE_RISOTTO_BASE_DIR-$BASE_NAME.build"
function dnf_opt_base() {
INSTALL_DIR=$1
echo "--setopt=install_weak_deps=False --nodocs --noplugins --installroot=$INSTALL_DIR --releasever $RELEASEVER"
echo "--setopt=install_weak_deps=False --setopt=fastestmirror=True --nodocs --noplugins --installroot=$INSTALL_DIR --releasever $RELEASEVER"
}
function dnf_opt() {
INSTALL_DIR=$1
INSTALL_PKG=$2
OPT=$(dnf_opt_base "$INSTALL_DIR")
OPT=$(dnf_opt_base "$INSTALL_DIR")
echo "$OPT install $INSTALL_PKG"
}
function new_package_base() {
if [ "$INSTALL_TOOL" = "dnf" ]; then
OPT=$(dnf_opt "$BASE_DIR" "$BASE_PKG")
OPT=$(dnf_opt "$TMP_BASE_DIR" "$BASE_PKG")
dnf --assumeno $OPT | grep ^" " > "$BASE_PKGS_FILE".new
else
debootstrap --include="$BASE_PKG" --variant=minbase "$RELEASEVER" "$BASE_DIR" > /dev/null
chroot "$BASE_DIR" dpkg-query -f '${binary:Package} ${source:Version}\n' -W > "$BASE_PKGS_FILE".new
debootstrap --include="$BASE_PKG" --variant=minbase "$RELEASEVER" "$TMP_BASE_DIR" >> /var/log/risotto/build_image.log
chroot "$TMP_BASE_DIR" dpkg-query -f '${binary:Package} ${source:Version}\n' -W > "$BASE_PKGS_FILE".new
fi
}
function install_base() {
if [ "$INSTALL_TOOL" = "dnf" ]; then
OPT=$(dnf_opt "$BASE_DIR" "$BASE_PKG")
OPT=$(dnf_opt "$TMP_BASE_DIR" "$BASE_PKG")
dnf --assumeyes $OPT
fi
}
function new_package() {
if [ "$INSTALL_TOOL" = "dnf" ]; then
OPT=$(dnf_opt_base "$IMAGE_NAME_RISOTTO_IMAGE_DIR")
dnf $OPT update
OPT=$(dnf_opt "$IMAGE_NAME_RISOTTO_IMAGE_DIR" "$PKG")
OPT=$(dnf_opt_base "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP")
dnf --assumeno $OPT update >> /var/log/risotto/build_image.log
OPT=$(dnf_opt "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP" "$PKG")
dnf --assumeno $OPT | grep ^" " > "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs.new
else
chroot "$IMAGE_NAME_RISOTTO_IMAGE_DIR" apt update > /dev/null 2>&1
chroot "$IMAGE_NAME_RISOTTO_IMAGE_DIR" apt install --no-install-recommends --yes $PKG -s 2>/dev/null|grep ^"Inst " > "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs.new
chroot "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP" apt update >> /var/log/risotto/build_image.log 2>&1
chroot "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP" apt install --no-install-recommends --yes $PKG -s 2>/dev/null|grep ^"Inst " > "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs.new
fi
}
function install_pkg() {
if [ "$INSTALL_TOOL" = "dnf" ]; then
OPT=$(dnf_opt "$IMAGE_NAME_RISOTTO_IMAGE_DIR" "$PKG")
OPT=$(dnf_opt "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP" "$PKG")
dnf --assumeyes $OPT
else
chroot "$IMAGE_NAME_RISOTTO_IMAGE_DIR" apt install --no-install-recommends --yes $PKG
chroot "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP" bash -c "export DEBIAN_FRONTEND=noninteractive; apt install --no-install-recommends --yes $PKG"
fi
}
if [ ! -f "$BASE_LOCK" ] || [ ! -f "$BASE_TAR" ]; then
if [ ! -f "$BASE_LOCK" ] || [ ! -d "$BASE_DIR" ]; then
echo " - reinstallation de l'image de base"
rm -rf "$BASE_DIR"
new_package_base
diff -u "$BASE_PKGS_FILE" "$BASE_PKGS_FILE".new && NEW_BASE=false || NEW_BASE=true
if [ ! -f "$BASE_TAR" ] || [ "$NEW_BASE" = true ]; then
if [ ! -d "$BASE_DIR" ] || [ "$NEW_BASE" = true ]; then
mkdir -p "$IMAGE_BASE_RISOTTO_BASE_DIR"
rm -rf "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP"
install_base
cd "$IMAGE_BASE_RISOTTO_BASE_DIR"
tar cf "$BASE_TAR" "$BASE_NAME"
cd - > /dev/null
if [ -f "$BASE_PKGS_FILE" ]; then
mv "$BASE_PKGS_FILE" "$BASE_PKGS_FILE".old
fi
mv "$BASE_PKGS_FILE".new "$BASE_PKGS_FILE"
rm -rf "$IMAGE_BASE_RISOTTO_BASE_DIR"
rm -rf "$BASE_DIR"
mv "$TMP_BASE_DIR" "$BASE_DIR"
fi
rm -rf "$BASE_DIR"
touch "$BASE_LOCK"
fi
tar xf "$BASE_TAR"
mv "$BASE_NAME" "$IMAGE_NAME_RISOTTO_IMAGE_DIR"
rm -rf "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP"
cp --reflink=auto -a "$BASE_DIR/" "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP"
if [ -n "$COPR" ]; then
#FIXME signature...
mkdir -p "$REPO_DIR"
@ -124,17 +118,18 @@ if [ -n "$COPR" ]; then
cd - > /dev/null
fi
if [ "$FUSION" = true ]; then
dnf -y install "https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$RELEASEVER.noarch.rpm" --installroot="$IMAGE_NAME_RISOTTO_IMAGE_DIR" > /dev/null
dnf -y install "https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$RELEASEVER.noarch.rpm" --installroot="$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP" >> /var/log/risotto/build_image.log
fi
# FIXME verifier s'il y a des modifs sur pre/post
if [ -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".base.pkgs ] && [ -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs ]; then
echo " - différence(s) avec les paquets de base"
diff -u "$IMAGE_NAME_RISOTTO_IMAGE_DIR".base.pkgs "$BASE_PKGS_FILE" && INSTALL=false || INSTALL=true
[ ! -f "$IMAGE_NAME_RISOTTO_IMAGE_NAME" ] && INSTALL=true
[ ! -d "$IMAGE_NAME_RISOTTO_IMAGE_DIR" ] && INSTALL=true
else
INSTALL=true
fi
new_package
if [ "$INSTALL" = false ]; then
echo " - différence(s) avec les paquets de l'image"
@ -146,26 +141,12 @@ if [ "$INSTALL" = false ]; then
fi
if [ "$INSTALL" = true ]; then
echo " - installation"
if [ -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR"_"$RELEASEVER".version ]; then
VERSION=$(cat "$IMAGE_NAME_RISOTTO_IMAGE_DIR"_"$RELEASEVER".version)
if [ -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".version ]; then
VERSION=$(cat "$IMAGE_NAME_RISOTTO_IMAGE_DIR".version)
else
VERSION=0
fi
mkdir "$RISOTTO_IMAGE_DIR/tmp"
ORI_DIR=$PWD
cd "$RISOTTO_IMAGE_DIR/tmp"
if [ ! "$VERSION" = 0 ] && [ -f "$IMAGE_NAME_RISOTTO_IMAGE_NAME" ]; then
tar xf "$IMAGE_NAME_RISOTTO_IMAGE_NAME"
# if [ "$INSTALL_TOOL" = "apt" ]; then
# chown _apt "$IMAGE_NAME"
# fi
# else
# mkdir "$IMAGE_NAME"
fi
#cd "$IMAGE_NAME"
make_changelog "$IMAGE_NAME" "$VERSION" "$OS_NAME" "$RELEASEVER" > "$IMAGE_NAME_RISOTTO_IMAGE_DIR"_"$RELEASEVER"_"$VERSION"_changelog.md
cd $ORI_DIR
rm -rf "$RISOTTO_IMAGE_DIR/tmp"
install_pkg
sleep 2
@ -173,25 +154,20 @@ if [ "$INSTALL" = true ]; then
. "$script"
done
CONTAINER=$IMAGE_NAME make_volatile /etc
ROOT=$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP make_volatile /etc
if [ ! "$?" = 0 ]; then
echo "make_volatile failed"
exit 1
fi
cd "$RISOTTO_IMAGE_DIR/$IMAGE_NAME"
if [ -f "$IMAGE_NAME_RISOTTO_IMAGE_NAME" ]; then
mv -f "$IMAGE_NAME_RISOTTO_IMAGE_NAME" "$IMAGE_NAME_RISOTTO_IMAGE_NAME".old
fi
tar cf "$IMAGE_NAME_RISOTTO_IMAGE_NAME" .
sha256sum "$IMAGE_NAME_RISOTTO_IMAGE_NAME" > "$IMAGE_NAME_RISOTTO_IMAGE_NAME".sha
cd - > /dev/null
cp -f "$BASE_PKGS_FILE" "$IMAGE_NAME_RISOTTO_IMAGE_DIR".base.pkgs
rm -rf "$IMAGE_NAME_RISOTTO_IMAGE_DIR"
mv "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP" "$IMAGE_NAME_RISOTTO_IMAGE_DIR"
cp --reflink=auto -f "$BASE_PKGS_FILE" "$IMAGE_NAME_RISOTTO_IMAGE_DIR".base.pkgs
mv -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs.new "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs
mv -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".md5sum.new "$IMAGE_NAME_RISOTTO_IMAGE_DIR".md5sum
VERSION=$((VERSION + 1))
echo "$VERSION" > "$IMAGE_NAME_RISOTTO_IMAGE_DIR"_"$RELEASEVER".version
echo "$VERSION" > "$IMAGE_NAME_RISOTTO_IMAGE_DIR".version
fi
rm -rf "$IMAGE_NAME_RISOTTO_IMAGE_DIR"
rm -rf "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP"
echo " => OK"
exit 0

22
ansible/sbin/compare_image Executable file
View file

@ -0,0 +1,22 @@
#!/bin/bash
SRV=$1
if [ -z "$SRV" ]; then
echo "usage: $0 machine"
exit 1
fi
dirname="/var/lib/risotto/templates/$SRV"
if [ ! -d "$dirname" ]; then
echo "cannot find $dirname"
echo "usage: $0 machine"
exit 1
fi
cd $dirname
find -type f | while read a; do
cfile="/var/lib/machines/$SRV/usr/share/factory/$a"
if [ -f "$cfile" ]; then
diff -u "$cfile" "$a"
fi
done
cd - > /dev/null

0
ansible/sbin/diagnose Normal file → Executable file
View file

5
ansible/sbin/make_volatile
View file

@ -1,9 +1,8 @@
#!/bin/bash -e
if [ -z $CONTAINER ]; then
echo "PAS DE CONTAINER"
if [ -z $ROOT]; then
echo "PAS DE ROOT"
exit 1
fi
ROOT="/var/lib/risotto/images/$CONTAINER"
echo "$ROOT"
DESTDIR="$ROOT/usr/lib/tmpfiles.d"
CONF_DST="/usr/share/factory"

18
ansible/sbin/update_images Normal file → Executable file
View file

@ -8,13 +8,15 @@ IMAGE_BASE_RISOTTO_BASE_DIR="$RISOTTO_IMAGE_DIR/image_bases"
rm -f $IMAGE_BASE_RISOTTO_BASE_DIR*.build
ls /var/lib/risotto/images_files/ | while read image; do
if [ -d /var/lib/risotto/images_files/"$image" ]; then
echo
echo "Install image $image"
/usr/local/sbin/build_image "$image"
fi
done
if [ -z "$1" ]; then
ls /var/lib/risotto/images_files/ | while read image; do
if [ -d /var/lib/risotto/images_files/"$image" ]; then
echo
echo "Install image $image"
/usr/local/sbin/build_image "$image" || true
fi
done
fi
#rm -f $IMAGE_BASE_RISOTTO_BASE_DIR*.build
MACHINES=""
@ -28,7 +30,7 @@ for nspawn in $(ls /etc/systemd/nspawn/*.nspawn); do
IMAGE_NAME_RISOTTO_IMAGE_NAME=${content##* }
diff -q "$IMAGE_NAME_RISOTTO_IMAGE_NAME".sha "$SHA_MACHINE" > /dev/null || (
echo "Reinstall machine $machine"
machinectl stop $machine
machinectl stop $machine || true
while true; do
machinectl status "$machine" > /dev/null 2>&1 || break
sleep 1

1
risotto.conf.example
View file

@ -1,3 +1,4 @@
[directories]
dataset = '/home/gnunux/git/risotto/dataset/seed'
dest = 'installations'
dest_templates = 'templates'

28
sbin/risotto_check_certificates Executable file
View file

@ -0,0 +1,28 @@
#!/usr/bin/env python3
from os import walk
from datetime import datetime
week_day = datetime.now().isocalendar().week
week_cert = f'certificate_{week_day}.crt'
for p, d, f in walk('pki/x509'):
if not d and not f:
print('empty dir, you can remove it: ', p)
if not f:
continue
if f == ['serial_number']:
continue
if not p.endswith('/ca') and not p.endswith('/server') and not p.endswith('/client'):
print('unknown directory: ', p)
continue
if week_cert in f:
continue
for ff in f:
if ff.startswith('certificate_') and ff.endswith('.crt'):
print(f'old certificat in: ', p)
break
else:
print('cannot find certificat in: ', p)

235
sbin/risotto_display Executable file
View file

@ -0,0 +1,235 @@
#!/usr/bin/env python3
from asyncio import run
from tabulate import tabulate
from argparse import ArgumentParser
from rougail.utils import normalize_family
from tiramisu.error import PropertiesOptionError
from risotto.machine import load, remove_cache, ROUGAIL_NAMESPACE
HIDE_SECRET = True
def list_to_string(lst):
if isinstance(lst, list):
return "\n".join([str(val) for val in lst])
return lst
async def get_files_subelements(type_name, element, files_subelement, files_cols):
data = {}
if not await element.option('activate').value.get():
return data
for subelement in files_subelement.values():
if subelement['type'] == 'subelement':
try:
value = list_to_string(await element.option(subelement['key']).value.get())
# FIXME except AttributeError:
except Exception:
value = ''
elif subelement['type'] == 'information':
value = await element.information.get(subelement['key'], '')
elif subelement['type'] == 'none':
value = subelement['value']
else:
raise Exception('unknown subelement')
if value != '':
files_cols.add(subelement['key'])
data[subelement['key']] = value
if type_name == 'overrides':
data['name'] = f'/systemd/system/{data["source"]}.d/rougail.conf'
if not data['engine']:
data['engine'] = 'none'
elif not data['engine']:
data['engine'] = 'cheetah'
return data
async def services(config, values):
files_subelement = {'Source': {'key': 'source', 'type': 'information'},
'Nom': {'key': 'name', 'type': 'subelement'},
'Variable': {'key': 'variable', 'type': 'subelement'},
'Propriétaire': {'key': 'owner', 'type': 'subelement'},
'Groupe': {'key': 'group', 'type': 'subelement'},
'Mode': {'key': 'mode', 'type': 'subelement'},
'Moteur': {'key': 'engine', 'type': 'information'},
}
disabled_services = []
for service in await config.option.list(type="all"):
doc = await service.option.doc()
files_lst = []
files_cols = set()
if not await service.option('manage').value.get():
doc += " - unmanaged"
if not await service.option('activate').value.get():
disabled_services.append([doc])
else:
for type in await service.list(type="all"):
type_name = await type.option.doc()
if type_name in ['files', 'overrides']:
for element in await type.list(type="all"):
data = await get_files_subelements(type_name, element, files_subelement, files_cols)
if data:
files_lst.append(data)
elif type_name == 'manage':
pass
elif type_name == 'activate':
if not await type.value.get():
doc += " - unactivated"
else:
print("FIXME " + type_name)
if files_lst:
keys = [key for key, val in files_subelement.items() if val['key'] in files_cols]
values[doc] = {'keys': keys, 'lst': []}
for lst in files_lst:
values[doc]['lst'].append([val for key, val in lst.items() if key in files_cols])
if disabled_services:
values["Services désactivés"] = {'keys': ['Nom'], 'lst': disabled_services}
async def table_leader(config, read_only):
keys = ['Description']
if read_only:
keys.append('Cachée')
leadership_lst = await config.list(type="all")
leader = leadership_lst.pop(0)
leader_owner = await leader.owner.get()
follower_names = [await follower.option.name() for follower in leadership_lst]
doc = await leader.option.doc()
properties = await leader.property.get()
if 'mandatory' in properties:
doc += '*'
name = await leader.option.name()
lst = [[f'{doc} ({name})']]
if read_only:
if 'hidden' in properties:
hidden = 'oui'
else:
hidden = ''
lst[0].append(hidden)
for idx, leader_value in enumerate(await leader.value.get()):
keys.append(f'Valeur {idx}')
keys.append(f'Utilisateur {idx}')
lst[0].append(leader_value)
lst[0].append(leader_owner)
for follower_idx, follower_name in enumerate(follower_names):
follower_option = config.option(follower_name, idx)
if idx == 0:
doc = await follower_option.option.doc()
properties = await follower_option.property.get()
if 'mandatory' in properties:
doc += '*'
name = await follower_option.option.name()
lst.append([f'{doc} ({name})'])
if read_only:
if 'hidden' in properties:
hidden = 'oui'
else:
hidden = ''
lst[-1].append(hidden)
try:
lst[follower_idx + 1].append(list_to_string(await follower_option.value.get()))
lst[follower_idx + 1].append(await follower_option.owner.get())
except PropertiesOptionError:
pass
# leader = next leader_iter
# if master_values is None:
# master_values = await subconfig.value.get()
return {'keys': keys, 'lst': lst}
async def table(config, prefix_len, values, read_only):
lst = []
for subconfig in await config.option.list(type="all"):
# prefix = prefix_len * 2 * ' '
# if await subconfig.option.isoptiondescription():
# prefix += '=>'
# else:
# prefix += '-'
# display_str = f'{prefix} {description}'
# if name != description:
# display_str = f'{display_str} ({name})'
name = await subconfig.option.name()
doc = await subconfig.option.doc()
if prefix_len == 0 and ROUGAIL_NAMESPACE != name:
doc = doc.capitalize()
if prefix_len == 0 and name == 'services':
values['Services'] = {}
await services(subconfig, values['Services'])
elif await subconfig.option.isoptiondescription():
od_name = f'{doc} ({(await subconfig.option.path()).split(".", 1)[1]})'
values[od_name] = None
if await subconfig.option.isleadership():
values[od_name] = await table_leader(subconfig, read_only)
else:
values[od_name] = await table(subconfig, prefix_len + 1, values, read_only)
else:
value = list_to_string(await subconfig.value.get())
doc = await subconfig.option.doc()
properties = await subconfig.property.get()
if 'mandatory' in properties:
doc += '*'
name = await subconfig.option.name()
lst.append([f'{doc} ({name})', value])
if read_only:
if 'hidden' in properties:
hidden = 'oui'
else:
hidden = ''
lst[-1].append(hidden)
lst[-1].append(await subconfig.owner.get())
keys = ['Description', 'Valeur']
if read_only:
keys.append('Cachée')
keys.append('Utilisateur')
return {'keys': keys, 'lst': lst}
async def main():
parser = ArgumentParser()
parser.add_argument('server_name')
parser.add_argument('--read_only', action='store_true')
parser.add_argument('--nocache', action='store_true')
args = parser.parse_args()
if args.nocache:
remove_cache()
values = {}
server_name = args.server_name
config = await load(hide_secret=HIDE_SECRET,
original_display_name=True,
valid_mandatories=args.read_only,
)
if not args.read_only:
await config.property.read_write()
root_option = config.option(normalize_family(server_name))
try:
await root_option.option.get()
except AttributeError:
exit(f'Unable to find {server_name} configuration: {[await o.option.name() for o in await config.option.list(type="optiondescription")]}')
await table(root_option, 0, values, args.read_only)
for title, dico in values.items():
if title == 'Services':
if not dico:
continue
print()
print(title)
print('=' * len(title))
print()
for subtitle, dic in dico.items():
print()
print(' ' + subtitle)
print(' ' + '-' * len(subtitle))
print()
print(tabulate(dic['lst'], headers=dic['keys'], tablefmt="fancy_grid"))
elif dico['lst']:
print()
print(title)
print('=' * len(title))
print()
print(tabulate(dico['lst'], headers=dico['keys'], tablefmt="fancy_grid"))
run(main())

31
sbin/risotto_templates Executable file
View file

@ -0,0 +1,31 @@
#!/usr/bin/env python3
from asyncio import run
from argparse import ArgumentParser
from traceback import print_exc
from risotto.machine import templates, remove_cache, load, INSTALL_DIR
async def main():
parser = ArgumentParser()
parser.add_argument('server_name')
parser.add_argument('--nocache', action='store_true')
parser.add_argument('--debug', action='store_true')
args = parser.parse_args()
if args.nocache:
remove_cache()
config = await load()
try:
await templates(args.server_name,
config,
)
except Exception as err:
if args.debug:
print_exc()
exit(err)
print(f'templates generated in {INSTALL_DIR} directory')
run(main())

415
src/risotto/image.py
View file

@ -1,64 +1,210 @@
from shutil import copy2, copytree, rmtree
from shutil import copy2, copytree
from os import listdir, makedirs
from os.path import join, isdir, isfile, dirname
from yaml import load as yaml_load, SafeLoader
from json import load as json_load
#
from rougail import RougailConfig # , RougailConvert
#
from .utils import RISOTTO_CONFIG, SERVERS, MULTI_FUNCTIONS
FUNCTIONS_FILE = 'funcs.py'
from .utils import RISOTTO_CONFIG
class ModuleCfg():
def __init__(self, module_name):
self.module_name = module_name
self.dictionaries_dir = []
self.modules = []
self.functions_file = [FUNCTIONS_FILE]
self.functions_file = []
self.templates_dir = []
self.patches_dir = []
self.extra_dictionaries = {}
self.servers = []
self.depends = []
self.manuals = []
self.tests = []
self.providers = []
self.suppliers = []
def __repr__(self):
return str(vars(self))
def list_applications() -> dict:
class Applications:
def __init__(self) -> None:
self.datasets = RISOTTO_CONFIG.get('directories', {}).get('datasets', ['dataset'])
self.application_directories = self._load_application_directories()
def _load_application_directories(self) -> dict:
"""List all service applications in datasets
Returns something link:
{<applicationservice>: seed/<applicationservice>}
"""
applications = {}
for dataset_directory in self.datasets:
for applicationservice in listdir(dataset_directory):
applicationservice_dir = join(dataset_directory, applicationservice)
if not isdir(applicationservice_dir) or \
not isfile(join(applicationservice_dir, 'applicationservice.yml')):
continue
if applicationservice in applications:
raise Exception(f'multi applicationservice: {applicationservice} ({applicationservice_dir} <=> {applications[applicationservice]})')
applications[applicationservice] = applicationservice_dir
return applications
class Modules:
"""Modules are defined by the end user
A module is the a list of service applications
The class collects all the useful information for the module
"""
{<applicationservice>: seed/<applicationservice>
"""
dataset_directory = RISOTTO_CONFIG['directories']['dataset']
applications = {}
for applicationservice in listdir(dataset_directory):
applicationservice_dir = join(dataset_directory, applicationservice)
if not isdir(applicationservice_dir):
continue
if applicationservice in applications:
raise Exception(f'multi applicationservice: {applicationservice} ({applicationservice_dir} <=> {applications[applicationservice]})')
applications[applicationservice] = applicationservice_dir
return applications
def __init__(self,
host_applicationsservices: list,
applicationservices: Applications,
applicationservice_provider: str,
modules_name: list,
modules: dict,
) -> None:
self.application_directories = applicationservices.application_directories
self.module_infos = {}
self.module_infos['host'] = self._load_module_informations('host',
host_applicationsservices,
is_host=True,
)
for module_name in modules_name:
if modules_name == 'host':
raise Exception('forbidden module name: "host"')
self.module_infos[module_name] = self._load_module_informations(module_name,
[applicationservice_provider] + modules[module_name],
is_host=False,
)
def get(self,
module_name: str,
) -> ModuleCfg:
return self.module_infos[module_name]
def _load_module_informations(self,
module_name: str,
applicationservices: list,
is_host: bool,
) -> ModuleCfg:
"""Create a ModuleCfg object and collect informations
A module must depend to an unique distribution
"""
cfg = ModuleCfg(module_name)
distribution = None
for applicationservice in applicationservices:
ret = self._load_applicationservice(applicationservice,
cfg,
)
if ret:
if distribution:
raise Exception(f'duplicate distribution for {cfg.module_name}: {distribution} and {ret} (dependencies: {cfg.depends}) ')
distribution = ret
if not is_host and not distribution:
raise Exception(f'cannot found any linux distribution for {module_name}')
return cfg
def _load_applicationservice(self,
appname: str,
cfg: ModuleCfg,
) -> str:
"""extract informations from an application service and load it's dependency
informations collected is store to the module
returns the name of current distribution, if found
"""
if appname not in self.application_directories:
raise Exception(f'cannot find application dependency "{appname}"')
cfg.depends.append(appname)
as_dir = self.application_directories[appname]
self._load_applicationservice_directories(as_dir,
cfg,
)
with open(join(as_dir, 'applicationservice.yml')) as yaml:
app = yaml_load(yaml, Loader=SafeLoader)
provider = app.get('provider')
if provider:
cfg.providers.setdefault(provider, [])
if appname not in cfg.providers[provider]:
cfg.providers[provider].append(appname)
supplier = app.get('supplier')
if supplier:
self.suppliers.setdefault(supplier, [])
if appname not in self.suppliers[supplier]:
self.suppliers[supplier].append(appname)
if 'distribution' in app and app['distribution']:
distribution = appname
else:
distribution = None
for depend in app.get('depends', []):
if depend in cfg.depends:
#this dependancy is already loaded for this module
continue
ret = self._load_applicationservice(depend,
cfg,
)
if ret:
if distribution:
raise Exception(f'duplicate distribution for {cfg.module_name}: {distribution} and {ret} (dependencies: {cfg.depends}) ')
distribution = ret
return distribution
def _load_applicationservice_directories(self,
as_dir: str,
cfg: ModuleCfg,
) -> None:
# dictionaries
dictionaries_dir = join(as_dir, 'dictionaries')
if isdir(dictionaries_dir):
cfg.dictionaries_dir.append(dictionaries_dir)
# funcs
funcs_dir = join(as_dir, 'funcs')
if isdir(funcs_dir):
for f in listdir(funcs_dir):
if f.startswith('__'):
continue
cfg.functions_file.append(join(funcs_dir, f))
# templates
templates_dir = join(as_dir, 'templates')
if isdir(templates_dir):
cfg.templates_dir.append(templates_dir)
# patches
patches_dir = join(as_dir, 'patches')
if isdir(patches_dir):
cfg.patches_dir.append(patches_dir)
# extras
extras_dir = join(as_dir, 'extras')
if isdir(extras_dir):
for extra in listdir(extras_dir):
extra_dir = join(extras_dir, extra)
if isdir(extra_dir):
cfg.extra_dictionaries.setdefault(extra, []).append(extra_dir)
# manual
for type in ['image', 'install']:
manual_dir = join(as_dir, 'manual')
if isdir(join(manual_dir, type)):
cfg.manuals.append(manual_dir)
break
# tests
tests_dir = join(as_dir, 'tests')
if isdir(tests_dir):
cfg.tests.append(tests_dir)
def applicationservice_copy(src_file: str,
dst_file: str,
copy_if_not_exists: bool,
) -> None:
if isdir(src_file):
if not isdir(dst_file):
makedirs(dst_file)
for subfilename in listdir(src_file):
if not copy_if_not_exists or not isfile(dst_file):
src = join(src_file, subfilename)
dst = join(dst_file, subfilename)
if isfile(src):
copy2(src, dst)
else:
copytree(src, dst)
elif not copy_if_not_exists or not isfile(dst_file):
#if not copy_if_not_exists or not isfile(dst_file):
src = join(src_file, subfilename)
dst = join(dst_file, subfilename)
if isfile(src):
copy2(src, dst)
else:
copytree(src, dst)
else:
dst = dirname(dst_file)
if not isdir(dst):
makedirs(dst)
@ -68,154 +214,6 @@ def applicationservice_copy(src_file: str,
copytree(src_file, dst_file)
def load_applicationservice_cfg(appname: str,
as_dir: str,
install_dir: str,
cfg: ModuleCfg,
copy_manual_dir: bool,
copy_tests: bool,
) -> None:
cfg.modules.append(appname)
# dictionaries
dictionaries_dir = join(as_dir, 'dictionaries')
if isdir(dictionaries_dir):
cfg.dictionaries_dir.append(dictionaries_dir)
# funcs
funcs_dir = join(as_dir, 'funcs')
if isdir(funcs_dir):
for f in listdir(funcs_dir):
if f.startswith('__'):
continue
cfg.functions_file.append(join(funcs_dir, f))
# templates
templates_dir = join(as_dir, 'templates')
if isdir(templates_dir):
cfg.templates_dir.append(templates_dir)
# extras
extras_dir = join(as_dir, 'extras')
if isdir(extras_dir):
for extra in listdir(extras_dir):
extra_dir = join(extras_dir, extra)
if isdir(extra_dir):
cfg.extra_dictionaries.setdefault(extra, []).append(extra_dir)
if copy_manual_dir:
# manual
for type in ['image', 'install']:
manual_dir = join(as_dir, 'manual', type)
if not isdir(manual_dir):
continue
for filename in listdir(manual_dir):
src_file = join(manual_dir, filename)
if type == 'image':
dst_file = join(install_dir, 'manual', filename)
copy_if_not_exists = False
else:
dst_file= join(install_dir, '..', filename)
copy_if_not_exists = True
applicationservice_copy(src_file,
dst_file,
copy_if_not_exists,
)
if copy_tests:
tests_dir = join(as_dir, 'tests')
if isdir(tests_dir):
for filename in listdir(tests_dir):
src_file = join(tests_dir, filename)
dst_file = join(install_dir, 'tests', filename)
applicationservice_copy(src_file,
dst_file,
False,
)
def load_applicationservice(appname: str,
install_dir: str,
cfg: ModuleCfg,
applications: dict,
copy_manual_dir: bool,
copy_tests: bool,
providers: dict,
suppliers: dict,
) -> None:
if appname not in applications:
raise Exception(f'cannot find application dependency "{appname}"')
as_dir = applications[appname]
applicationservice_file = join(as_dir, 'applicationservice.yml')
if not isfile(applicationservice_file):
raise Exception(f'cannot find application service file "{applicationservice_file}"')
load_applicationservice_cfg(appname,
as_dir,
install_dir,
cfg,
copy_manual_dir,
copy_tests,
)
cfg.depends.append(appname)
with open(applicationservice_file) as yaml:
app = yaml_load(yaml, Loader=SafeLoader)
provider = app.get('provider')
if provider:
providers.setdefault(provider, [])
if appname not in providers[provider]:
providers[provider].append(appname)
supplier = app.get('supplier')
if supplier:
suppliers.setdefault(supplier, [])
if appname not in suppliers[supplier]:
suppliers[supplier].append(appname)
if 'distribution' in app and app['distribution']:
distribution = appname
else:
distribution = None
for xml in app.get('depends', []):
if xml in cfg.depends:
continue
ret = load_applicationservice(xml,
install_dir,
cfg,
applications,
copy_manual_dir,
copy_tests,
providers,
suppliers,
)
if ret:
if distribution:
raise Exception(f'duplicate distribution for {cfg.module_name}: {distribution} and {ret} (dependencies: {cfg.depends}) ')
distribution = ret
return distribution
def load_image_informations(module_name: str,
install_dir: str,
datas: dict,
applications: dict,
copy_manual_dir: bool,
copy_tests: bool,
providers: dict,
suppliers: dict,
) -> ModuleCfg:
cfg = ModuleCfg(module_name)
distribution = None
for applicationservice in datas['applicationservices']:
ret = load_applicationservice(applicationservice,
install_dir,
cfg,
applications,
copy_manual_dir,
copy_tests,
providers,
suppliers,
)
if ret:
if distribution:
raise Exception(f'duplicate distribution for {cfg.module_name}: {distribution} and {ret} (dependencies: {cfg.depends}) ')
distribution = ret
if module_name != 'host' and not distribution:
raise Exception(f'cannot found any linux distribution for {module_name}')
return cfg
async def valid_mandatories(config):
mandatories = await config.value.mandatory()
if mandatories:
@ -232,60 +230,3 @@ async def valid_mandatories(config):
# await value_pprint(await config.value.dict(), config)
exit(1)
#raise Exception('configuration has mandatories variables without values')
def load_config(copy_manual_dir=False,
copy_tests=False,
clean_directories=False,
):
module_infos = {}
applications = list_applications()
with open('servers.json', 'r') as server_fh:
jsonfile = json_load(server_fh)
SERVERS.update(jsonfile['servers'])
modules = jsonfile['modules']
for module_name, datas in modules.items():
providers = {}
suppliers = {}
install_dir = join(RISOTTO_CONFIG['directories']['dest'], module_name)
if clean_directories:
if isdir(install_dir):
rmtree(install_dir)
makedirs(install_dir)
module_infos[module_name] = {'infos': load_image_informations(module_name,
install_dir,
datas,
applications,
copy_manual_dir,
copy_tests,
providers,
suppliers,
),
'providers': providers,
'suppliers': suppliers,
'install_dir': install_dir,
}
return module_infos
#
#
def load_module_config(module_name: str,
module_info: dict,
):
cfg = RougailConfig.copy()
cfg['variable_namespace'] = ROUGAIL_NAMESPACE
cfg['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
if module_name == 'host':
#FIXME server_name == host ?
#FIXME cfg['tmpfile_dest_dir'] = datas['values'][f'{ROUGAIL_NAMESPACE}.host_install_dir'] + '/host/configurations/host'
cfg['default_systemd_directory'] = '/usr/local/lib/systemd'
cfg['templates_dir'] = module_info['infos'].templates_dir
cfg['dictionaries_dir'] = module_info['infos'].dictionaries_dir
cfg['functions_file'] = module_info['infos'].functions_file
cfg['multi_functions'] = MULTI_FUNCTIONS
cfg['extra_dictionaries'] = module_info['infos'].extra_dictionaries
cfg['extra_annotators'] = ['risotto.rougail']
cfg['internal_functions'] = list(FUNCTIONS.keys())
cfg['force_convert_dyn_option_description'] = True
cfg['module_name'] = module_name
#cfg['patches_dir'] = join(test_dir, 'patches')
return cfg

381
src/risotto/machine.py
View file

@ -1,16 +1,19 @@
from .utils import SERVERS, SERVERS_JSON, MULTI_FUNCTIONS, load_domains
from .image import load_config, valid_mandatories # , load_modules_rougail_config
from rougail import RougailConfig, RougailConvert
from .utils import MULTI_FUNCTIONS, load_zones, value_pprint, RISOTTO_CONFIG
from .image import Applications, Modules, valid_mandatories, applicationservice_copy
from .rougail.annotator import calc_providers, calc_providers_global, calc_providers_dynamic, calc_providers_dynamic_follower, calc_providers_follower
from os.path import isfile
from rougail import RougailConfig, RougailConvert
from os import remove, makedirs, listdir
from os.path import isfile, isdir, abspath
from json import dump as json_dump, load as json_load
from yaml import load as yaml_load, SafeLoader
#
from tiramisu import Config
from .utils import value_pprint
from tiramisu import Config, valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal, calc_value
from rougail.utils import normalize_family
from rougail import RougailSystemdTemplate
#
#
from shutil import rmtree
def tiramisu_display_name(kls,
dyn_name: 'Base'=None,
suffix: str=None,
@ -23,49 +26,77 @@ def tiramisu_display_name(kls,
return name
async def set_values(server_name, config, datas):
if 'values' not in datas:
return
server_path = normalize_family(server_name)
for vpath, value in datas['values'].items():
path = f'{server_path}.{vpath}'
try:
if isinstance(value, dict):
for idx, val in value.items():
await config.option(path, int(idx)).value.set(val)
else:
await config.option(path).value.set(value)
except Exception as err:
await value_pprint(await config.value.dict(), config)
error_msg = f'cannot configure variable {vpath} for server "{server_name}": {err}'
raise Exception(error_msg) from err
def get_ip_from_domain(domain):
if not domain:
return
hostname, domainname = domain.split('.', 1)
return DOMAINS[domainname][1][DOMAINS[domainname][0].index(hostname)]
return optiondescription['option_0']
CONFIG_FILE = 'servers.yml'
ROUGAIL_NAMESPACE = 'general'
ROUGAIL_NAMESPACE_DESCRIPTION = 'Général'
FUNCTIONS = {'get_ip_from_domain': get_ip_from_domain,
'calc_providers': calc_providers,
TIRAMISU_CACHE = 'tiramisu_cache.py'
VALUES_CACHE = 'values_cache.json'
INFORMATIONS_CACHE = 'informations_cache.json'
INSTALL_DIR = RISOTTO_CONFIG['directories']['dest']
INSTALL_TEMPLATES_DIR = RISOTTO_CONFIG['directories']['dest_templates']
FUNCTIONS = {'calc_providers': calc_providers,
'calc_providers_global': calc_providers_global,
'calc_providers_dynamic': calc_providers_dynamic,
'calc_providers_dynamic_follower': calc_providers_dynamic_follower,
'calc_providers_follower': calc_providers_follower,
'valid_network_netmask': valid_network_netmask,
'valid_ip_netmask': valid_ip_netmask,
'valid_broadcast': valid_broadcast,
'valid_in_network': valid_in_network,
'valid_not_equal': valid_not_equal,
'calc_value': calc_value,
'normalize_family': normalize_family,
}
def re_create(dirname):
if isdir(dirname):
rmtree(dirname)
makedirs(dirname)
def remove_cache():
if isfile(TIRAMISU_CACHE):
remove(TIRAMISU_CACHE)
if isfile(VALUES_CACHE):
remove(VALUES_CACHE)
if isfile(INFORMATIONS_CACHE):
remove(INFORMATIONS_CACHE)
async def templates(server_name,
config,
templates_informations,
just_copy=False,
):
engine = RougailSystemdTemplate(config, templates_informations)
subconfig = config.option(normalize_family(server_name))
try:
await subconfig.option.get()
except:
servers = [await server.option.description() for server in await config.option.list('optiondescription')]
raise Exception(f'cannot find server name "{server_name}": {servers}')
rougailconfig = RougailConfig.copy()
rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE
rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
rougailconfig['tmp_dir'] = 'tmp'
if not just_copy:
rougailconfig['destinations_dir'] = INSTALL_DIR
else:
rougailconfig['destinations_dir'] = INSTALL_TEMPLATES_DIR
rougailconfig['templates_dir'] = await subconfig.information.get('templates_dir')
rougailconfig['patches_dir'] = await subconfig.information.get('patches_dir')
rougailconfig['functions_file'] = await subconfig.information.get('functions_files')
is_host = await subconfig.information.get('module') == 'host'
if is_host:
host_install_dir = f'{ROUGAIL_NAMESPACE}.host_install_dir'
rougailconfig['tmpfile_dest_dir'] = await subconfig.option(host_install_dir).value.get()
rougailconfig['default_systemd_directory'] = '/usr/local/lib/systemd'
else:
rougailconfig['tmpfile_dest_dir'] = '/usr/local/lib'
rougailconfig['default_systemd_directory'] = '/systemd'
re_create(rougailconfig['destinations_dir'])
re_create(rougailconfig['tmp_dir'])
engine = RougailSystemdTemplate(subconfig, rougailconfig)
if just_copy:
# for all engine to none
ori_engines = {}
@ -79,95 +110,211 @@ async def templates(server_name,
except Exception as err:
print()
print(f'=== Configuration: {server_name} ===')
values = await config.value.dict()
await value_pprint(values, config)
print(err)
print(await config.option('general.nginx.nginx_default_http').value.get())
values = await subconfig.value.dict()
await value_pprint(values, subconfig)
raise err from err
if just_copy:
for eng, old_engine in ori_engines.items():
engine.engines[eng] = old_engine
async def load(cache_file,
cache_values,
clean_directories=False,
copy_manual_dir=False,
copy_tests=False,
hide_secret=False,
):
display_name=tiramisu_display_name
#load_zones()
# # load images
#FIXME useful
module_infos = load_config(copy_manual_dir,
copy_tests,
clean_directories,
)
# modules_rougail_config = load_modules_rougail_config(module_infos)
cfg = RougailConfig.copy()
cfg['variable_namespace'] = ROUGAIL_NAMESPACE
cfg['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
cfg['multi_functions'] = MULTI_FUNCTIONS
cfg['extra_annotators'] = ['risotto.rougail']
cfg['internal_functions'] = list(FUNCTIONS.keys())
cfg['force_convert_dyn_option_description'] = True
# cfg['module_name'] = module_name
functions_files = set()
load_domains()
for server_name, datas in SERVERS.items():
module_info = module_infos[datas['module']]
functions_files |= set(module_info['infos'].functions_file)
cfg['functions_file'] = list(functions_files)
if not isfile(cache_file):
eolobj = RougailConvert(cfg)
class Loader:
def __init__(self,
cache_file,
cache_values,
cache_informations,
clean_directories,
hide_secret,
original_display_name,
valid_mandatories,
config_file=CONFIG_FILE,
):
self.cache_file = cache_file
self.cache_values = cache_values
self.cache_informations = cache_informations
self.hide_secret = hide_secret
self.original_display_name = original_display_name
self.valid_mandatories = valid_mandatories
self.config_file = config_file
if clean_directories:
if isdir(INSTALL_DIR):
rmtree(INSTALL_DIR)
makedirs(INSTALL_DIR)
def before(self):
with open(self.config_file, 'r') as server_fh:
self.servers_json = yaml_load(server_fh, Loader=SafeLoader)
cfg = RougailConfig.copy()
cfg['variable_namespace'] = ROUGAIL_NAMESPACE
cfg['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
cfg['multi_functions'] = MULTI_FUNCTIONS
cfg['extra_annotators'] = ['risotto.rougail']
cfg['internal_functions'] = list(FUNCTIONS.keys())
cfg['force_convert_dyn_option_description'] = True
cfg['risotto_globals'] = {}
for server_name, datas in SERVERS.items():
module_info = module_infos[datas['module']]
cfg['dictionaries_dir'] = module_info['infos'].dictionaries_dir
cfg['extra_dictionaries'] = module_info['infos'].extra_dictionaries
informations = SERVERS_JSON['servers'][server_name].get('informations')
if informations:
cfg['risotto_globals'][server_name] = {'global:server_name': server_name,
'global:zones_name': informations['zones_name'],
'global:zones_list': list(range(len(informations['zones_name']))),
}
values = []
for s_idx in cfg['risotto_globals'][server_name]['global:zones_list']:
if not s_idx:
values.append(server_name)
else:
values.append(informations['extra_domainnames'][s_idx - 1])
cfg['risotto_globals'][server_name]['global:server_names'] = values
else:
cfg['risotto_globals'][server_name] = {'global:server_name': server_name}
cfg['risotto_globals'][server_name]['global:module_name'] = datas['module']
eolobj.load_dictionaries(path_prefix=server_name)
tiram_obj = eolobj.save(cache_file)
else:
with open(cache_file) as fh:
tiram_obj = fh.read()
optiondescription = FUNCTIONS.copy()
try:
exec(tiram_obj, None, optiondescription)
except Exception as err:
print(tiram_obj)
raise Exception(f'unknown error when load tiramisu object {err}') from err
config = await Config(optiondescription['option_0'],
display_name=display_name,
)
if not isfile(cache_values):
rougail = RougailConvert(cfg)
self.templates_dir = {}
self.patches_dir = {}
functions_files = set()
self.functions_files = {}
applicationservices = Applications()
zones = self.servers_json['zones']
self.modules = {}
for host_name, datas in self.servers_json['hosts'].items():
modules_name = {mod_datas['module'] for mod_datas in datas['servers'].values()}
modules = Modules(datas['applicationservices'],
applicationservices,
datas['applicationservice_provider'],
modules_name,
self.servers_json['modules']
)
module_info = modules.get('host')
cfg['risotto_globals'][host_name] = {'global:server_name': host_name,
'global:module_name': 'host',
'global:host_install_dir': abspath(INSTALL_DIR),
}
functions_files |= set(module_info.functions_file)
self.load_dictionaries(cfg, module_info, host_name, rougail)
modules_info = {}
for server_name, server_datas in datas['servers'].items():
module_info = modules.get(server_datas['module'])
zones_name = server_datas['informations']['zones_name']
values = [f'{server_name}.{zones[zone_name]["domain_name"]}' for zone_name in zones_name]
cfg['risotto_globals'][values[0]] = {'global:host_name': host_name,
'global:server_name': values[0],
'global:server_names': values,
'global:zones_name': zones_name,
'global:zones_list': list(range(len(zones_name))),
'global:module_name': server_datas['module'],
}
server_datas['server_name'] = values[0]
functions_files |= set(module_info.functions_file)
self.load_dictionaries(cfg, module_info, values[0], rougail)
modules_info[module_info.module_name] = module_info.depends
self.modules[host_name] = modules_info
cfg['functions_file'] = list(functions_files)
self.tiram_obj = rougail.save(self.cache_file)
def load_dictionaries(self, cfg, module_info, server_name, rougail):
cfg['dictionaries_dir'] = module_info.dictionaries_dir
cfg['extra_dictionaries'] = module_info.extra_dictionaries
cfg['functions_file'] = module_info.functions_file
rougail.load_dictionaries(path_prefix=server_name)
self.templates_dir[server_name] = module_info.templates_dir
self.patches_dir[server_name] = module_info.patches_dir
self.functions_files[server_name] = module_info.functions_file
async def load(self):
optiondescription = FUNCTIONS.copy()
try:
exec(self.tiram_obj, None, optiondescription)
except Exception as err:
print(self.tiram_obj)
raise Exception(f'unknown error when load tiramisu object {err}') from err
if self.original_display_name:
display_name = None
else:
display_name = tiramisu_display_name
self.config = await Config(optiondescription['option_0'],
display_name=display_name,
)
async def after(self):
config = self.config
await config.property.pop('validator')
await config.property.pop('cache')
for server_name, datas in SERVERS.items():
await set_values(server_name, config, datas)
load_zones(self.servers_json)
await config.information.set('zones', self.servers_json['zones'])
for host_name, hosts_datas in self.servers_json['hosts'].items():
information = config.option(normalize_family(host_name)).information
await information.set('module', 'host')
await information.set('templates_dir', self.templates_dir[host_name])
await information.set('patches_dir', self.patches_dir[host_name])
await information.set('functions_files', self.functions_files[host_name])
await self.set_values(host_name, config, hosts_datas)
for datas in hosts_datas['servers'].values():
server_name = datas['server_name']
information = config.option(normalize_family(server_name)).information
await information.set('module', datas['module'])
await information.set('templates_dir', self.templates_dir[server_name])
await information.set('patches_dir', self.patches_dir[server_name])
await information.set('functions_files', self.functions_files[server_name])
await self.set_values(server_name, config, datas)
# FIXME only one host_name is supported
await config.information.set('modules', self.modules[host_name])
# await config.information.set('modules', {module_name: module_info.depends for module_name, module_info in self.module_infos.items() if module_name in modules})
await config.property.read_only()
await config.property.add('cache')
await valid_mandatories(config)
with open(cache_values, 'w') as fh:
if self.valid_mandatories:
await valid_mandatories(config)
with open(self.cache_values, 'w') as fh:
json_dump(await config.value.exportation(), fh)
with open(self.cache_informations, 'w') as fh:
json_dump(await config.information.exportation(), fh)
async def set_values(self,
server_name,
config,
datas,
):
if 'values' not in datas:
return
server_path = normalize_family(server_name)
await config.owner.set(self.config_file)
for vpath, value in datas['values'].items():
path = f'{server_path}.{vpath}'
try:
if isinstance(value, dict):
for idx, val in value.items():
await config.option(path, int(idx)).value.set(val)
else:
await config.option(path).value.set(value)
except Exception as err:
await value_pprint(await config.value.dict(), config)
error_msg = f'cannot configure variable {vpath} for server "{server_name}": {err}'
raise Exception(error_msg) from err
await config.owner.set('user')
async def finish(self):
await self.config.property.read_only()
class LoaderCache(Loader):
def before(self):
with open(self.cache_file) as fh:
self.tiram_obj = fh.read()
async def after(self):
with open(self.cache_values, 'r') as fh:
await self.config.value.importation(json_load(fh))
with open(self.cache_informations, 'r') as fh:
informations = json_load(fh)
# null is not a valid key in json => 'null'
informations[None] = informations.pop('null')
await self.config.information.importation(informations)
async def load(clean_directories=False,
hide_secret=False,
original_display_name: bool=False,
valid_mandatories: bool=True,
):
if isfile(TIRAMISU_CACHE) and isfile(VALUES_CACHE) and isfile(INFORMATIONS_CACHE):
loader_obj = LoaderCache
else:
with open(cache_values, 'r') as fh:
await config.value.importation(json_load(fh))
await config.property.read_only()
return module_infos, cfg, config
loader_obj = Loader
loader = loader_obj(TIRAMISU_CACHE,
VALUES_CACHE,
INFORMATIONS_CACHE,
clean_directories,
hide_secret,
original_display_name,
valid_mandatories,
)
loader.before()
await loader.load()
await loader.after()
await loader.finish()
return loader.config

95
src/risotto/rougail/annotator.py
View file

@ -1,10 +1,12 @@
from rougail.annotator.variable import Walk
from rougail.error import DictConsistencyError
from risotto.utils import _, multi_function
from warnings import warn
def _parse_kwargs(provider, dns, kwargs, index=None):
if not isinstance(dns, list):
raise Exception('pfff')
values = {}
for key, value in kwargs.items():
if '_' not in key:
@ -14,7 +16,15 @@ def _parse_kwargs(provider, dns, kwargs, index=None):
for idx, data in values.items():
if index is not None and int(idx) != index:
continue
if 'dns' not in data or (isinstance(data['dns'], list) and dns not in data['dns']) or (not isinstance(data['dns'], list) and data['dns'] != dns):
if 'dns' not in data:
continue
if isinstance(data['dns'], list):
for ddns in data['dns']:
if ddns in dns:
break
else:
continue
elif data['dns'] not in dns:
continue
del data['dns']
yield data
@ -142,7 +152,13 @@ class Annotator(Walk):
continue
nf_dns = variable.path.split('.', 1)[0]
server_name = self.objectspace.space.variables[nf_dns].doc
self.suppliers.setdefault(variable.supplier, []).append({'option': variable, 'dns': server_name, 'path_prefix': nf_dns, 'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'])})
self.suppliers.setdefault(variable.supplier, []).append({'option': variable,
'dns': server_name,
'path_prefix': nf_dns,
'server_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names'],
'zone_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'],
'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'])
})
def convert_suppliers(self):
for supplier, data in self.suppliers.items():
@ -152,10 +168,16 @@ class Annotator(Walk):
if supplier not in self.providers:
continue
for p_dico in self.providers[supplier]:
if s_dico['zones'] & p_dico['zones']:
s_dico['option'].value = p_dico['dns']
common_zones = s_dico['zones'] & p_dico['zones']
if common_zones:
for idx, zone in enumerate(p_dico['zone_names']):
if zone in common_zones:
break
dns = p_dico['server_names'][idx]
# dns = p_dico["dns"]
s_dico['option'].value = dns
new_value = self.objectspace.value(None)
new_value.name = p_dico['dns']
new_value.name = dns
s_dico['option'].value = [new_value]
break
@ -168,6 +190,10 @@ class Annotator(Walk):
nf_dns = variable.path.split('.', 1)[0]
server_name = self.objectspace.space.variables[nf_dns].doc
provider_name = variable.provider
if self.objectspace.rougailconfig['risotto_globals'][server_name]['global:module_name'] == 'host':
server_names = [server_name]
else:
server_names = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names']
if ':' in provider_name:
key_name, key_type = provider_name.rsplit(':', 1)
is_provider = False
@ -175,7 +201,13 @@ class Annotator(Walk):
key_name = key_type = provider_name
is_provider = True
if provider_name != 'Host':
self.providers.setdefault(provider_name, []).append({'option': variable, 'dns': server_name, 'path_prefix': nf_dns, 'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'])})
self.providers.setdefault(provider_name, []).append({'option': variable,
'dns': server_name,
'path_prefix': nf_dns,
'server_names': server_names,
'zone_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'],
'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name']),
})
if key_name != 'global' and key_name not in self.suppliers:
#warn(f'cannot find supplier "{key_name}" for "{server_name}"')
continue
@ -234,10 +266,12 @@ class Annotator(Walk):
if key_name != 'global':
param = self.objectspace.param(variable.xmlfiles)
param.name = 'dns'
param.text = server_name
param.text = server_names
fill.param.append(param)
if key_name == 'global':
param = self.objectspace.param(variable.xmlfiles)
if provider_name not in self.objectspace.rougailconfig['risotto_globals'][server_name]:
raise DictConsistencyError(f'cannot find provider "{provider_name}" for variable "{variable.name}"', 200, variable.xmlfiles)
param.text = self.objectspace.rougailconfig['risotto_globals'][server_name][provider_name]
param.name = 'value'
fill.param.append(param)
@ -296,48 +330,3 @@ class Annotator(Walk):
if not hasattr(self.objectspace.space.variables[nf_dns].constraints, 'fill'):
self.objectspace.space.variables[nf_dns].constraints.fill = []
self.objectspace.space.variables[nf_dns].constraints.fill.append(fill)
# def convert_get_linked_information(self):
# if not hasattr(self.objectspace.space, 'constraints') or \
# not hasattr(self.objectspace.space.constraints, 'fill'):
# return
# for fill in self.objectspace.space.constraints.fill:
# if fill.name == 'get_linked_configuration':
# # add server_name
# param = self.objectspace.param(fill.xmlfiles)
# param.name = 'server_name'
# param.type = 'information'
# param.text = 'server_name'
# fill.param.append(param)
# # add current_user
# param = self.objectspace.param(fill.xmlfiles)
# param.name = 'current_user'
# param.type = 'information'
# param.text = 'current_user'
# fill.param.append(param)
# # add test
# param = self.objectspace.param(fill.xmlfiles)
# param.name = 'test'
# param.type = 'target_information'
# param.text = 'test'
# fill.param.append(param)
#
# def convert_provider(self):
# if not hasattr(self.objectspace.space, 'variables'):
# return
# for family in self.get_families():
# if not hasattr(family, 'provider'):
# continue
# if 'dynamic' not in vars(family):
# raise Exception(_(f'{family.name} is not a dynamic family so cannot have provider attribute'))
# if not hasattr(family, 'information'):
# family.information = self.objectspace.information(family.xmlfiles)
# family.information.provider = family.provider
# del family.provider
# for variable in self.get_variables():
# if not hasattr(variable, 'provider'):
# continue
# if not hasattr(variable, 'information'):
# variable.information = self.objectspace.information(variable.xmlfiles)
# variable.information.provider = variable.provider
# del variable.provider

79
src/risotto/utils.py
View file

@ -1,18 +1,11 @@
from os import environ
from json import load
from typing import List
from ipaddress import ip_address
from toml import load as toml_load
from pprint import pprint
SETTINGS = {'config': None}
MULTI_FUNCTIONS = []
DOMAINS = {}
ZONES = {}
SERVERS_JSON = {}
SERVERS = {}
CONFIGS = {}
with open(environ.get('CONFIG_FILE', 'risotto.conf'), 'r') as fh:
@ -40,68 +33,20 @@ async def value_pprint(dico, config):
pprint(pprint_dict)
def load_zones_server():
if 'zones' in SERVERS_JSON:
return
with open('servers.json', 'r') as server_fh:
SERVERS_JSON.update(load(server_fh))
def load_zones():
global ZONES
if ZONES:
return
load_zones_server()
ZONES.update(SERVERS_JSON['zones'])
for server_name, server in SERVERS_JSON['servers'].items():
if 'informations' not in server:
continue
server_zones = server['informations']['zones_name']
server_extra_domainnames = server['informations'].get('extra_domainnames', [])
if len(server_zones) > 1 and len(server_zones) != len(server_extra_domainnames) + 1:
raise Exception(f'the server "{server_name}" has more that one zone, please set correct number of extra_domainnames ({len(server_zones) - 1} instead of {len(server_extra_domainnames)})')
for idx, zone_name in enumerate(server_zones):
zone_domain_name = ZONES[zone_name]['domain_name']
if idx == 0:
zone_server_name = server_name
else:
zone_server_name = server_extra_domainnames[idx - 1]
server_domain_name = zone_server_name.split('.', 1)[1]
if zone_domain_name and zone_domain_name != server_domain_name:
raise Exception(f'wrong server_name "{zone_server_name}" in zone "{zone_name}" should ends with "{zone_domain_name}"')
ZONES[zone_name].setdefault('hosts', []).append(server_name)
def load_domains():
global DOMAINS
if DOMAINS:
return
load_zones()
for zone_name, zone in SERVERS_JSON['zones'].items():
if 'domain_name' in zone:
hosts = []
ips = []
for host in ZONES[zone_name].get('hosts', []):
hosts.append(host.split('.', 1)[0])
ips.append(_get_ip(host, [zone_name], 0))
DOMAINS[zone['domain_name']] = (tuple(hosts), tuple(ips))
def load_zones(servers_json):
zones = servers_json['zones']
for host_name, hosts in servers_json['hosts'].items():
for server_name, server in hosts['servers'].items():
server_zones = server['informations']['zones_name']
for idx, zone_name in enumerate(server_zones):
zone = zones[zone_name]
zone.setdefault('hosts', {})
zone['hosts'][server_name] = _get_ip(server_name, zone)
def _get_ip(server_name: str,
zones_name: List[str],
index: str,
zone: dict,
) -> str:
if server_name is None or zones_name is None:
return
load_zones()
index = int(index)
zone_name = zones_name[index]
if zone_name not in ZONES:
raise ValueError(f"cannot set IP in unknown zone '{zone_name}'")
zone = ZONES[zone_name]
if server_name not in zone['hosts']:
raise ValueError(f"cannot set IP in unknown server '{server_name}'")
server_index = zone['hosts'].index(server_name)
# FIXME make a cache, machine should not change IP
server_index = len(zone['hosts'])
return str(ip_address(zone['start_ip']) + server_index)