simplify ansible role

ansible/action_plugins/build_images.py

@@ -8,51 +8,37 @@ from ansible.plugins.action import ActionBase
 from risotto.utils import RISOTTO_CONFIG
 
 
+
+
 class ActionModule(ActionBase):
     def run(self, tmp=None, task_vars=None):
         super(ActionModule, self).run(tmp, task_vars)
         module_args = self._task.args.copy()
         modules = module_args['modules']
+        copy_tests = module_args.get('copy_tests', False)
         dataset_directories = RISOTTO_CONFIG['directories']['datasets']
         install_dir = join('/tmp/risotto/images')
         if isdir(install_dir):
             rmtree(install_dir)
+        if copy_tests:
+            install_tests_dir = join('/tmp/risotto/tests')
+            if isdir(install_tests_dir):
+                rmtree(install_tests_dir)
         for module_name, depends in modules.items():
             for dataset_directory in dataset_directories:
                 for depend in depends:
-                    manual = join(dataset_directory, depend, 'manual', 'image')
+                    if copy_tests:
-                    if not isdir(manual):
+                        tests_dir = join(dataset_directory, depend, 'tests')
-                        continue
+                        if isdir(tests_dir):
-                    for filename in listdir(manual):
+                            for filename in listdir(tests_dir):
-                        src_file = join(manual, filename)
+                                src_file = join(tests_dir, filename)
-                        dst_file = join(install_dir, module_name, filename)
+                                dst_file = join(install_tests_dir, module_name, filename)
-                        if isdir(src_file):
+                                copy(src_file, dst_file)
-                            if not isdir(dst_file):
+#                    manual = join(dataset_directory, depend, 'manual', 'image')
-                                makedirs(dst_file)
+#                    if not isdir(manual):
-                            for subfilename in listdir(src_file):
+#                        continue
-                                if not isfile(dst_file):
+#                    for filename in listdir(manual):
-                                    src = join(src_file, subfilename)
+#                        src_file = join(manual, filename)
-                                    dst = join(dst_file, subfilename)
+#                        dst_file = join(install_dir, module_name, filename)
-                                    if isfile(src):
+#                        copy(src_file, dst_file)
-                                        copy2(src, dst)
-                                    else:
-                                        copytree(src, dst)
-                        elif not isfile(dst_file):
-                            dst = dirname(dst_file)
-                            if not isdir(dst):
-                                makedirs(dst)
-                            if isfile(src_file):
-                                copy2(src_file, dst_file)
-                            else:
-                                copytree(src_file, dst_file)
         return dict(ansible_facts=dict({}))
-#A REFAIRE ICI    tests_dir = join(as_dir, 'tests')
-#A REFAIRE ICI    if isdir(tests_dir):
-#A REFAIRE ICI        cfg.tests.append(tests_dir)
-#A REFAIRE ICI#        for filename in listdir(tests_dir):
-#A REFAIRE ICI#            src_file = join(tests_dir, filename)
-#A REFAIRE ICI#            dst_file = join(INSTALL_DIR, 'tests', filename)
-#A REFAIRE ICI#            applicationservice_copy(src_file,
-#A REFAIRE ICI#                                    dst_file,
-#A REFAIRE ICI#                                    False,
-#A REFAIRE ICI#                                    )

ansible/action_plugins/rougail.py

@@ -1,8 +1,11 @@
 #!/usr/bin/python3
 from asyncio import run
-from os import readlink
+from os import readlink, walk, chdir, getcwd, makedirs
-from os.path import join, islink
+from os.path import join, islink, isdir
-from risotto.machine import load, templates
+from risotto.machine import load, templates, INSTALL_DIR, INSTALL_CONFIG_DIR, INSTALL_TMPL_DIR, INSTALL_IMAGES_DIR, INSTALL_TESTS_DIR
+from rougail.utils import normalize_family
+from shutil import rmtree
+import tarfile
 try:
     from ansible.plugins.action import ActionBase
     from ansible.module_utils.basic import AnsibleModule
@@ -10,59 +13,192 @@ try:
         def __init__(self):
             pass
 except:
-    import traceback
-    traceback.print_exc()
     class ActionBase():
         def __init__(self, *args, **kwargs):
             raise Exception('works only with ansible')
 
 
-async def build_files(server_name: str,
+ARCHIVES_DIR = '/tmp/new_configurations'
+
+
+async def build_files(hostname: str,
+                      only_machine: str,
                       just_copy: bool,
+                      copy_tests: bool,
                       ) -> None:
-    config = await load()
+    config = await load(copy_tests=copy_tests)
-    await templates(server_name,
+    if only_machine:
+        machines = [only_machine]
+    else:
+        machines = [await subconfig.option.description() for subconfig in await config.option.list(type='optiondescription')]
+#    shasums = {}
+    directories = {}
+    for machine in machines:
+        if just_copy and hostname == machine:
+            continue
+        await templates(machine,
                         config,
                         just_copy=just_copy,
+                        copy_manuals=True,
                         )
+        #FIXME dest_dir?
+
+        is_host = machine == hostname
+#        shasums[machine] = {'shasums': get_shasums(dest_dir, is_host)}
+        if is_host:
+#            shasums[machine]['config_dir'] = '/usr/local/lib'
+            directories[machine] = '/usr/local/lib'
+        else:
+#            shasums[machine]['config_dir'] = await config.option(normalize_family(machine)).option('general.config_dir').value.get()
+            directories[machine] = await config.option(normalize_family(machine)).option('general.config_dir').value.get()
+    return directories
+
+
+def is_diff(server_name, remote_directories):
+    ret = {}
+    module = FakeModule()
+    current_path = getcwd()
+    root = join(INSTALL_DIR, INSTALL_CONFIG_DIR, server_name)
+    chdir(root)
+    search_paths = [join(directory[2:], f) for directory, subdirectories, files in walk('.') for f in files]
+    chdir(current_path)
+    for path in search_paths:
+        if path not in remote_directories:
+            return True
+        full_path = join(root, path)
+        if not islink(full_path):
+            if remote_directories[path] != module.digest_from_file(full_path, 'sha256'):
+                return True
+        elif remote_directories[path] != readlink(full_path):
+            return True
+        remote_directories.pop(path)
+    if remote_directories:
+        return True
+    return False
 
 
 class ActionModule(ActionBase):
     def run(self, tmp=None, task_vars=None):
         super(ActionModule, self).run(tmp, task_vars)
         module_args = self._task.args.copy()
-        root_local = module_args.pop('root_local')
+        hostname = module_args.pop('hostname')
-        root_remote= module_args.pop('root_remote')
+        only_machine = module_args.pop('only_machine')
-        name = module_args.pop('hostname')
+        configure_host = module_args.pop('configure_host')
-        is_host = module_args.pop('is_host')
+        copy_tests = module_args.pop('copy_tests')
-        just_copy = module_args.get('just_copy', False)
+        if 'copy_templates' in module_args:
-        module_args['root'] = root_remote
+            copy_templates = module_args.pop('copy_templates')
-
-        run(build_files(name, just_copy))
-        #
-        remote = self._execute_module(module_name='compare', module_args=module_args, task_vars=task_vars)
-        if remote.get('failed'):
-            raise Exception(f'error in remote action: {remote["module_stdout"]}')
-        #
-        module = FakeModule()
-        modified_files = []
-        changed = False
-        for path in module_args['paths']:
-            full_path = join(root_local, path['name'][1:])
-            if remote['compare'].get(path['name']):
-                if remote['compare'][path['name']]['type'] == 'file':
-                    if remote['compare'][path['name']]['shasum'] == module.digest_from_file(full_path, 'sha256'):
-                        continue
         else:
-                    # it's a symlink
+            copy_templates = False
-                    if islink(full_path) and remote['compare'][path['name']]['name'] == readlink(full_path):
+        directories = run(build_files(hostname,
+                                       only_machine,
+                                       False,
+                                       copy_tests,
+                                       ))
+        module_args['directories'] = list(directories.values())
+        module_args['directories'].append('/var/lib/risotto/images_files')
+        remote = self._execute_module(module_name='compare',
+                                      module_args=module_args,
+                                      task_vars=task_vars,
+                                      )
+        if remote.get('failed'):
+            if 'module_stdout' in remote:
+                msg = remote['module_stdout']
+            else:
+                msg = remote['msg']
+            raise Exception(f'error in remote action: {msg}')
+        if copy_templates:
+            run(build_files(hostname,
+                            only_machine,
+                            True,
+                            copy_tests,
+                            ))
+
+        machines_changed = []
+        for machine, directory in directories.items():
+            if directory not in remote['directories']:
+                machines_changed.append(machine)
                 continue
-            changed = True
+            if is_diff(machine, remote['directories'][directory]):
-            modified_files.append(path['name'])
+                machines_changed.append(machine)
-        if not is_host:
+        current_path = getcwd()
-            for old_file in remote['old_files']:
+        if isdir(ARCHIVES_DIR):
-                changed = True
+            rmtree(ARCHIVES_DIR)
-    #            module_args['path'] = old_file
+        makedirs(ARCHIVES_DIR)
-    #            module_args['state'] = 'absent'
+        if machines_changed:
-    #            self._execute_module(module_name='ansible.builtin.file', module_args=module_args, task_vars=task_vars)
+            self._execute_module(module_name='file',
-        return dict(ansible_facts=dict({}), changed=changed)
+                                 module_args={'path': ARCHIVES_DIR,
+                                              'state': 'absent',
+                                              },
+                                 task_vars=task_vars,
+                                 )
+            self._execute_module(module_name='file',
+                                 module_args={'path': ARCHIVES_DIR,
+                                              'state': 'directory',
+                                              },
+                                 task_vars=task_vars,
+                                 )
+            machines = machines_changed.copy()
+            if self._task.args['hostname'] in machines_changed:
+                machine = self._task.args['hostname']
+                machines.remove(machine)
+                chdir(f'{task_vars["host_install_dir"]}/{INSTALL_CONFIG_DIR}/{machine}')
+                tar_filename = f'{ARCHIVES_DIR}/host.tar'
+                with tarfile.open(tar_filename, 'w') as archive:
+                    archive.add('.')
+                chdir(current_path)
+                self._transfer_file(tar_filename, tar_filename)
+
+            # archive and send
+            if machines:
+                chdir(f'{task_vars["host_install_dir"]}/{INSTALL_CONFIG_DIR}')
+                tar_filename = f'{ARCHIVES_DIR}/machines.tar'
+                with tarfile.open(tar_filename, 'w') as archive:
+                    for machine in machines:
+                        if machine == self._task.args['hostname']:
+                            continue
+                        archive.add(f'{machine}')
+                self._transfer_file(tar_filename, tar_filename)
+        else:
+            machines = []
+        # archive and send
+        chdir(f'{task_vars["host_install_dir"]}/{INSTALL_IMAGES_DIR}/')
+        tar_filename = f'{ARCHIVES_DIR}/{INSTALL_IMAGES_DIR}.tar'
+        with tarfile.open(tar_filename, 'w') as archive:
+            archive.add('.')
+        self._transfer_file(tar_filename, tar_filename)
+        # tests
+        self._execute_module(module_name='file',
+                             module_args={'path': '/var/lib/risotto/tests',
+                                          'state': 'absent',
+                                          },
+                             task_vars=task_vars,
+                             )
+        if copy_tests:
+            chdir(f'{task_vars["host_install_dir"]}/{INSTALL_TESTS_DIR}/')
+            tar_filename = f'{ARCHIVES_DIR}/{INSTALL_TESTS_DIR}.tar'
+            with tarfile.open(tar_filename, 'w') as archive:
+                archive.add('.')
+            self._transfer_file(tar_filename, tar_filename)
+        # templates
+        self._execute_module(module_name='file',
+                             module_args={'path': '/var/lib/risotto/templates',
+                                          'state': 'absent',
+                                          },
+                             task_vars=task_vars,
+                             )
+        if copy_templates:
+            chdir(f'{task_vars["host_install_dir"]}/')
+            tar_filename = f'{ARCHIVES_DIR}/{INSTALL_TMPL_DIR}.tar'
+            with tarfile.open(tar_filename, 'w') as archive:
+                archive.add(INSTALL_TMPL_DIR)
+            self._transfer_file(tar_filename, tar_filename)
+            remote = self._execute_module(module_name='unarchive',
+                                          module_args={'remote_src': True,
+                                                       'src': '/tmp/new_configurations/templates.tar',
+                                                       'dest': '/var/lib/risotto',
+                                                       },
+                                          task_vars=task_vars,
+                                          )
+        chdir(current_path)
+        changed = machines_changed != []
+        return dict(ansible_facts=dict({}), changed=changed, machines_changed=machines, host_changed=self._task.args['hostname'] in machines_changed)

ansible/host.yml

@@ -8,113 +8,9 @@
     update_cache: yes
     state: latest
 
-- name: "Build host files"
+- name: "Host is modified"
-  rougail:
+  include_tasks: host_modified.yml
-    paths: "{{ vars[inventory_hostname]['services'] | fileslist(is_host=True) }}"
+  when: build_host.host_changed
-    root_local: "{{ host_install_dir }}"
-    root_remote: "/"
-    hostname: "{{ inventory_hostname }}"
-    is_host: True
-
-- name: "Create /usr/local/lib/systemd/system"
-  file:
-    path: /usr/local/lib/systemd/system
-    state: directory
-    mode: 0755
-
-- name: "Copy service file only if not exists"
-  when: item.value['manage'] and item.value['activate'] and item.value['doc'].endswith('.service') and not item.value['doc'].endswith('@.service') and item.value['engine'] and item.value['engine'] != 'none'
-  copy:
-    src: '{{ host_install_dir }}/usr/local/lib/systemd/system/{{ item.value["doc"] }}'
-    force: no
-    dest: '/usr/local/lib/systemd/system/{{ item.value["doc"] }}'
-  loop: "{{ vars[inventory_hostname]['services'] | dict2items }}"
-  loop_control:
-    label: "{{ item.value['doc'] }}"
-
-- name: "Stop services"
-  when: item.value['manage'] and item.value['activate'] and item.value['doc'].endswith('.service') and not item.value['doc'].endswith('@.service') and item.value['engine'] != 'none'
-  ansible.builtin.service:
-    name: "{{ item.value['doc'] }}"
-    state: stopped
-  loop: "{{ vars[inventory_hostname]['services'] | dict2items }}"
-  loop_control:
-    label: "{{ item.value['doc'] }}"
-
-- name: "Create host directories"
-  file: path={{ item }} state=directory mode=0755
-  loop: "{{ vars[inventory_hostname]['services'] | directorieslist }}"
-
-- name: "Copy systemd-tmpfiles"
-  when: item.name.startswith('/usr/local/lib/risotto-tmpfiles.d')
-  ansible.builtin.copy:
-    src: "{{ host_install_dir }}/{{ item.name }}"
-    dest: "{{ item.name }}"
-    owner: "{{ item.owner }}"
-    group: "{{ item.group }}"
-    mode: "{{ item.mode }}"
-  loop: "{{ vars[inventory_hostname]['services'] | fileslist(is_host=True) }}"
-  loop_control:
-    label: "{{ item.name}}"
-
-- name: "Execute systemd-tmpfiles"
-  when: item.name.startswith('/usr/local/lib/risotto-tmpfiles.d')
-  command: /usr/bin/systemd-tmpfiles --create --clean --remove {{ item.name }}
-  loop: "{{ vars[inventory_hostname]['services'] | fileslist(is_host=True) }}"
-  loop_control:
-    label: "{{ item.name}}"
-
-- name: "Copy host files"
-  when: not item.name.startswith('/usr/local/lib/tmpfiles.d')
-  ansible.builtin.copy:
-    src: "{{ host_install_dir }}/{{ item.name }}"
-    dest: "{{ item.name }}"
-    owner: "{{ item.owner }}"
-    group: "{{ item.group }}"
-    mode: "{{ item.mode }}"
-  loop: "{{ vars[inventory_hostname]['services'] | fileslist(is_host=True) }}"
-  loop_control:
-    label: "{{ item.name}}"
-
-- name: "Reload systemd services configuration"
-  ansible.builtin.systemd:
-    daemon_reload: yes
-
-- name: "Enable services"
-  when: item.value['manage'] and item.value['activate'] and '@.service' not in item.value['doc']
-  ansible.builtin.service:
-    name: "{{ item.value['doc'] }}"
-    enabled: yes
-  loop: "{{ vars[inventory_hostname]['services'] | dict2items }}"
-  loop_control:
-    label: "{{ item.value['doc'] }}"
-
-- name: "Disable services"
-  when: item.value['manage'] and not item.value['activate'] and not item.value['undisable'] and '@.service' not in item.value['doc']
-  ansible.builtin.service:
-    name: "{{ item.value['doc'] }}"
-    enabled: yes
-  loop: "{{ vars[inventory_hostname]['services'] | dict2items }}"
-  loop_control:
-    label: "{{ item.value['doc'] }}"
-
-- name: "Start services"
-  when: item.value['manage'] and item.value['activate'] and item.value['doc'].endswith('.service') and not item.value['doc'].endswith('@.service') and item.value['engine'] != 'none'
-  ansible.builtin.service:
-    name: "{{ item.value['doc'] }}"
-    state: started
-  loop: "{{ vars[inventory_hostname]['services'] | dict2items }}"
-  loop_control:
-    label: "{{ item.value['doc'] }}"
-
-- name: "Restart services"
-  when: item.value['manage'] and item.value['activate'] and item.value['doc'].endswith('.service') and not item.value['doc'].endswith('@.service') and item.value['engine'] == 'none'
-  ansible.builtin.service:
-    name: "{{ item.value['doc'] }}"
-    state: restarted
-  loop: "{{ vars[inventory_hostname]['services'] | dict2items }}"
-  loop_control:
-    label: "{{ item.value['doc'] }}"
 
 - name: "Copy machines scripts"
   ansible.builtin.copy:
@@ -125,43 +21,39 @@
     mode: "0755"
   loop: "{{ lookup('fileglob', 'sbin/*', wantlist=True) | list }}"
 
-# Images informations
-- name: "Remove images tar"
-  local_action:
-    module: file
-    path: /tmp/risotto/images.tar
-    state: absent
-
-- name: "Build images files"
-  local_action:
-    module: build_images
-    modules: "{{ vars['modules'] }}"
-
-- name: "Compress images files"
-  local_action:
-    module: archive
-    path: "/tmp/risotto/images/"
-    dest: /tmp/risotto/images.tar
-    format: tar
-
 - name: "Remove dest images files"
   file:
     path: /var/lib/risotto/images_files
-    state: absent
+    state: "{{ item }}"
-
-- name: "Create images files"
-  file:
-    path: /var/lib/risotto/images_files
-    state: directory
     mode: "0700"
+  with_items:
+    - absent
+    - directory
 
 - name: "Copy images files"
   unarchive:
-    src: "/tmp/risotto/images.tar"
+    remote_src: true
+    src: "/tmp/new_configurations/images_files.tar"
     dest: "/var/lib/risotto/images_files"
 
 - name: "Create versions directory"
   file:
-    path: /var/lib/risotto/machines_versions
+    path: /var/lib/risotto/machines_informations
     state: directory
     mode: "0700"
+
+- name: "Empty tests files"
+  file:
+    path: /var/lib/risotto/tests
+    state: "{{ item }}"
+    mode: "0700"
+  with_items:
+    - absent
+    - directory
+    
+- name: "Copy tests files"
+  unarchive:
+    remote_src: true
+    src: "/tmp/new_configurations/tests.tar"
+    dest: "/var/lib/risotto/tests"
+  when: copy_tests

ansible/host_modified.yml

@@ -0,0 +1,74 @@
+- name: "Stop services"
+  ansible.builtin.service:
+    name: "{{ item.value['doc'] }}"
+    state: stopped
+  when: item.value['manage'] and item.value['activate'] and item.value['doc'].endswith('.service') and not item.value['doc'].endswith('@.service') and item.value['engine'] != 'none' and item.value['doc'] in services
+  loop: "{{ vars[inventory_hostname]['services'] | dict2items }}"
+  loop_control:
+    label: "{{ item.value['doc'] }}"
+
+- name: "Remove old config files"
+  file:
+    path: /usr/local/lib/
+    state: "{{ item }}"
+    mode: "0700"
+  with_items:
+    - absent
+    - directory
+
+- name: "Copy config files"
+  unarchive:
+    remote_src: true
+    src: "/tmp/new_configurations/host.tar"
+    dest: /usr/local/lib/
+    owner: root
+    group: root
+
+- name: "Execute systemd-tmpfiles"
+  command: /usr/bin/systemd-tmpfiles --create --clean --remove -E --exclude-prefix=/tmp
+
+- name: "Remove tmpfiles files directory"
+  local_action:
+    module: file
+    path: /usr/local/lib/tmpfiles.d/
+    state: absent
+
+- name: "Reload systemd services configuration"
+  ansible.builtin.systemd:
+    daemon_reload: yes
+
+- name: "Enable services"
+  when: item.value['manage'] and item.value['activate'] and '@.service' not in item.value['doc']
+  ansible.builtin.service:
+    name: "{{ item.value['doc'] }}"
+    enabled: yes
+  loop: "{{ vars[inventory_hostname]['services'] | dict2items }}"
+  loop_control:
+    label: "{{ item.value['doc'] }}"
+
+- name: "Disable services"
+  when: item.value['manage'] and not item.value['activate'] and not item.value['undisable'] and '@.service' not in item.value['doc']
+  ansible.builtin.service:
+    name: "{{ item.value['doc'] }}"
+    enabled: no
+  loop: "{{ vars[inventory_hostname]['services'] | dict2items }}"
+  loop_control:
+    label: "{{ item.value['doc'] }}"
+
+- name: "Start services"
+  when: item.value['manage'] and item.value['activate'] and item.value['doc'].endswith('.service') and not item.value['doc'].endswith('@.service') and item.value['engine'] != 'none'
+  ansible.builtin.service:
+    name: "{{ item.value['doc'] }}"
+    state: started
+  loop: "{{ vars[inventory_hostname]['services'] | dict2items }}"
+  loop_control:
+    label: "{{ item.value['doc'] }}"
+
+- name: "Restart services"
+  when: item.value['manage'] and item.value['activate'] and item.value['doc'].endswith('.service') and not item.value['doc'].endswith('@.service') and item.value['engine'] == 'none'
+  ansible.builtin.service:
+    name: "{{ item.value['doc'] }}"
+    state: restarted
+  loop: "{{ vars[inventory_hostname]['services'] | dict2items }}"
+  loop_control:
+    label: "{{ item.value['doc'] }}"

ansible/installations

@@ -1 +0,0 @@
-/home/gnunux/git/risotto/risotto/installations/

ansible/inventory.json

@@ -1,15 +0,0 @@
-{
-    "_meta": {
-        "hostvars": {}
-    },
-    "all": {
-        "children": [
-            "ungrouped"
-        ]
-    },
-    "ungrouped": {
-        "hosts": [
-            "cloud.silique.fr"
-        ]
-    }
-}

ansible/inventory.py

@@ -103,7 +103,8 @@ class RisottoInventory(object):
         ret['delete_old_image'] = False
         ret['configure_host'] = True
         ret['only_machine'] = None
-        ret['copy_template'] = False
+        ret['copy_templates'] = False
+        ret['copy_tests'] = False
         ret['host_install_dir'] = ret[host_name].pop('host_install_dir')
         return dumps(ret, cls=RougailEncoder)
 
@@ -117,7 +118,7 @@ async def main():
     except Exception as err:
         if DEBUG:
             print_exc()
-        exit(err)
+        print(err)
 
 
 run(main())

ansible/library/compare.py

@@ -1,10 +1,8 @@
 #!/usr/bin/python3
 
 from time import sleep
-from os import fdopen, walk, readlink
+from os import fdopen, walk, readlink, chdir, getcwd
-from os.path import join, islink
+from os.path import join, islink, isdir
-from dbus import SystemBus, Array
-from dbus.exceptions import DBusException
 
 from ansible.module_utils.basic import AnsibleModule
 
@@ -12,8 +10,8 @@ from ansible.module_utils.basic import AnsibleModule
 def run_module():
     # define available arguments/parameters a user can pass to the module
     module_args = dict(
-        root=dict(type='str', required=True),
+            #        shasums=dict(type='dict', required=True),
-        paths=dict(type='list', required=True),
+        directories=dict(type='list', required=True),
     )
 
     # seed the result dict in the object
@@ -22,10 +20,7 @@ def run_module():
     # state will include any data that you want your module to pass back
     # for consumption, for example, in a subsequent task
     result = dict(
-        changed=False,
+        directories={},
-        compare={},
-        symlink={},
-        old_files=[],
     )
 
     # the AnsibleModule object will be our abstraction working with Ansible
@@ -34,28 +29,48 @@ def run_module():
     # supports check mode
     module = AnsibleModule(
         argument_spec=module_args,
-        supports_check_mode=True
+        supports_check_mode=True,
     )
 
-    root = module.params['root']
+    current_path = getcwd()
-    if root != '/':
+    for directory in module.params['directories']:
-        paths = {join(root, path['name'][1:]): path['name'] for path in module.params['paths']}
+        result['directories'][directory] = {}
-        search_paths = [join(directory, f) for directory, subdirectories, files in walk(root) for f in files]
+        if not isdir(directory):
-    else:
+            continue
-        paths = {path['name']: path['name'] for path in module.params['paths']}
+        chdir(directory)
-        search_paths = paths
+        search_paths = [join(directory_[2:], f) for directory_, subdirectories, files in walk('.') for f in files]
         for path in search_paths:
-        if path in paths:
+            full_path = join(directory, path)
-            if not islink(path):
+            if not islink(full_path):
-                result['compare'][paths[path]] = {'type': 'file',
+                result['directories'][directory][path] = module.digest_from_file(full_path, 'sha256')
-                                                  'shasum': module.digest_from_file(path, 'sha256'),
-                                                  }
             else:
-                result['compare'][paths[path]] = {'type': 'symlink',
+                result['directories'][directory][path] = readlink(full_path)
-                                                  'name': readlink(path),
+    chdir(current_path)
-                                                  }
+#    current_path = getcwd()
-        else:
+#    for server_name, dico in module.params['shasums'].items():
-            result['old_files'].append(path)
+#        root = dico['config_dir']
+#        if not isdir(root):
+#            result['machines_changed'].append(server_name)
+#            continue
+#        chdir(root)
+#        search_paths = [join(directory[2:], f) for directory, subdirectories, files in walk('.') for f in files]
+#        chdir(current_path)
+#        for path in search_paths:
+#            if path in dico['shasums']:
+#                full_path = join(root, path)
+#                if not islink(full_path):
+#                    if module.digest_from_file(full_path, 'sha256') != dico['shasums'][path]:
+#                        result['machines_changed'].append(server_name)
+#                        break
+#                elif dico['shasums'][path] != readlink(full_path):
+#                    result['machines_changed'].append(server_name)
+#                    break
+#                del dico['shasums'][path]
+#            else:
+#                result['machines_changed'].append(server_name)
+#                break
+#        if server_name not in result['machines_changed'] and dico['shasums']:
+#            result['machines_changed'].append(server_name)
 
     module.exit_json(**result)
 

ansible/machine.yml

@@ -1,121 +1,15 @@
 - name: "Create SRV directory for {{ item.name}}"
+  file:
+    path: /var/lib/risotto/srv/{{ item.name }}
+    state: directory
+    mode: 0755
   when: "item.srv"
-  file: path=/var/lib/risotto/srv/{{ item.name }} state=directory mode=0755
 
-- name: "Create SystemD directory for {{ item.name }}"
+- name: "Create journald directory for {{ item.name }}"
-  file: path=/var/lib/risotto/journals/{{ item.name }} state=directory mode=0755
+  file:
-
+    path: /var/lib/risotto/journals/{{ item.name }}
-- name: "Build machine files for {{ item.name }}"
-  rougail:
-    paths: "{{ vars[item.name]['services'] | fileslist }}"
-    root_local: "{{ host_install_dir }}"
-    root_remote: "/var/lib/risotto/configurations/{{ item.name }}"
-    hostname: "{{ item.name}}"
-    is_host: False
-  register: up_to_date_configuration
-
-- name: "Change secrets right"
-  local_action:
-    module: file
-    path: "{{ host_install_dir }}/secrets"
     state: directory
-    mode: 0700
+    mode: 0755
 
-- name: "Compress files for {{ item.name }}"
+- name: "Create informations for {{ item.name }}"
-  local_action:
+  ansible.builtin.shell: "/usr/bin/echo {{ vars | modulename(item.name) }} > /var/lib/risotto/machines_informations/{{ item.name }}.image"
-    module: archive
-    path: "{{ host_install_dir }}/"
-    dest: /tmp/new_configurations/{{ item.name }}
-    format: tar
-  when: up_to_date_configuration.changed
-
-- name: "Build machine templates for {{ item.name }}"
-  rougail:
-    paths: "{{ vars[item.name]['services'] | fileslist }}"
-    root_local: "{{ host_install_dir }}"
-    root_remote: "/var/lib/risotto/configurations/{{ item.name }}"
-    hostname: "{{ item.name}}"
-    just_copy: true
-    is_host: False
-  when: copy_template
-  register: up_to_date_configuration
-
-- name: "Compress templates for {{ item.name }}"
-  local_action:
-    module: archive
-    path: "../templates/"
-    dest: /tmp/new_templates/{{ item.name }}
-    format: tar
-  when: copy_template
-
-- name: "Remove templates directory for {{ item.name }}"
-  file:
-    path: "/var/lib/risotto/templates/{{ item.name }}"
-    state: absent
-  when: copy_template
-
-- name: "Create templates directory for {{ item.name }}"
-  file:
-    path: "/var/lib/risotto/templates/{{ item.name }}"
-    state: directory
-  when: copy_template
-
-- name: "Copy templates for {{ item.name }}"
-  unarchive:
-    src: "/tmp/new_templates/{{ item.name }}"
-    dest: "/var/lib/risotto/templates/{{ item.name }}/"
-  when: copy_template
-
-- name: "Remove old image {{ vars | modulename(item.name) }}"
-  file:
-    path: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}"
-    state: absent
-  when: delete_old_image == true
-
-- name: "Stop machine {{ item.name }}"
-  machinectl:
-    state: stopped
-    machines: "{{ item.name }}"
-  when: delete_old_image == true
-
-- name: "Remove old machine {{ item.name }}"
-  file:
-    path: /var/lib/machines/{{ item.name }}
-    state: absent
-  when: delete_old_image == true
-
-- name: "Create system directory for {{ item.name }}"
-  file:
-    path: /var/lib/machines/{{ item.name }}
-    state: directory
-  register: system_directory_created
-
-- name: "Check image for {{ item.name }}"
-  stat:
-    path: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}"
-    follow: true
-  register: register_name
-  when: system_directory_created.changed
-
-#- name: Print return information from the previous task
-#  ansible.builtin.debug:
-#    var: register_name
-
-- name: "Build image for {{ item.name }}"
-  ansible.builtin.shell: "/usr/local/sbin/build_image {{ vars | modulename(item.name) }}"
-  when: "'stat' in register_name and not register_name.stat.exists"
-  register: ret
-  failed_when: ret.rc != 0
-
-- name: "Copy machine image for {{ item.name }}"
-  ansible.builtin.shell: "/usr/bin/cp -a --reflink=auto /var/lib/risotto/images/{{ vars | modulename(item.name) }}/* /var/lib/machines/{{ item.name }}"
-  when: system_directory_created.changed
-
-- name: "Copy machine image version for {{ item.name }}"
-  ansible.builtin.copy:
-    src: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}.version"
-    remote_src: true
-    dest: "/var/lib/risotto/machines_versions/{{ item.name }}.version"
-    owner: "root"
-    group: "root"
-  when: system_directory_created.changed

ansible/machines.yml

@@ -1,27 +1,30 @@
-- name: "Stop machines with new configuration"
+#- name: Print return information from the previous task
+#  ansible.builtin.debug:
+#    var: build_host.machines_changed
+
+- name: "Rebuild images"
+  ansible.builtin.shell: "/usr/local/sbin/update_images just_need_images"
+  register: ret
+  failed_when: ret.rc != 0
+
+- name: "Stop machines with new configuration {{ build_host.machines_changed }}"
   machinectl:
     state: stopped
-    machines: "{{ lookup('fileglob', '/tmp/new_configurations/*', wantlist=True) | map('basename') | list }}"
+    machines: "{{ build_host.machines_changed }}"
 
 - name: "Remove files directory"
   file:
     path: "/var/lib/risotto/configurations/{{ item }}"
     state: absent
-  loop: "{{ lookup('fileglob', '/tmp/new_configurations/*', wantlist=True) | map('basename') | list }}"
+  loop: "{{ build_host.machines_changed }}"
-
-- name: "Create files directory"
-  file:
-    path: "/var/lib/risotto/configurations/{{ item }}"
-    state: directory
-  loop: "{{ lookup('fileglob', '/tmp/new_configurations/*', wantlist=True) | map('basename') | list }}"
 
 - name: "Copy configuration"
   unarchive:
-    src: "{{ item }}"
+    src: /tmp/new_configurations/machines.tar
-    dest: /var/lib/risotto/configurations/{{ item | basename }}/
+    dest: /var/lib/risotto/configurations/
     owner: root
     group: root
-  loop: "{{ lookup('fileglob', '/tmp/new_configurations/*', wantlist=True) }}"
+  when: build_host.machines_changed
 
 - name: "Enable machines"
   machinectl:
@@ -38,9 +41,3 @@
     module: file
     path: /tmp/new_configurations
     state: absent
-
-- name: "Remove compressed templates directory"
-  local_action:
-    module: file
-    path: /tmp/new_templates
-    state: absent

ansible/playbook.txt

@@ -1,23 +0,0 @@
-    - name: installation dépendances
-      apt:
-        pkg:
-          - systemd-container
-          - dnf
-          - jq
-          - debootstrap
-          - htop
-          - gettext
-          - patch
-          - unzip
-          - mlocate
-          - xz-utils
-          - iptables
-        update_cache: yes
-        state: latest
-
-MARCHE
-    - name: installation dépendances
-      apt:
-        pkg: "{{ packages }}"
-        update_cache: yes
-        state: latest

ansible/playbook.yml

@@ -2,41 +2,28 @@
 - name: Risotto
   hosts: all
   tasks:
+    - name: "Build host files"
+      rougail:
+        hostname: "{{ vars['inventory_hostname'] }}"
+        only_machine: "{{ only_machine }}"
+        configure_host: "{{ configure_host }}"
+        copy_tests: "{{ copy_tests }}"
+        copy_templates: "{{ copy_templates }}"
+      register: build_host
+
     - name: "Configure the host"
       include_tasks: host.yml
       when: configure_host == true
 
-    - name: "Remove compressed files directory"
-      local_action:
-        module: file
-        path: /tmp/new_configurations
-        state: absent
-    
-    - name: "Create compressed configuration files directory"
-      local_action:
-        module: file
-        path: /tmp/new_configurations
-        state: directory
-        mode: 0700
-    
-    - name: "Remove compressed templates files directory"
-      local_action:
-        module: file
-        path: /tmp/new_templates
-        state: absent
-      when: copy_template
-    
-    - name: "Create compressed templates files directory"
-      local_action:
-        module: file
-        path: /tmp/new_templates
-        state: directory
-        mode: 0700
-      when: copy_template
-    
     - name: "Prepare machine configuration"
       include_tasks: machine.yml
+      when: item.name in build_host.machines_changed
       loop: "{{ vars | machineslist(only=only_machine) }}"
-    
+        #
+        #    - name: "Remove images"
+        #      include_tasks: remove_image.yml
+        #      loop: "{{ vars | machineslist(only=only_machine) }}"
+        #      when: delete_old_image == true
+        #
     - name: "Install and apply configurations"
       include_tasks: machines.yml

ansible/remove_image.yml

@@ -0,0 +1,14 @@
+- name: "Stop machine {{ item.name }}"
+  machinectl:
+    state: stopped
+    machines: "{{ item.name }}"
+
+- name: "Remove old machine {{ item.name }}"
+  file:
+    path: /var/lib/machines/{{ item.name }}
+    state: absent
+
+- name: "Remove old image {{ vars | modulename(item.name) }}"
+  file:
+    path: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}"
+    state: absent

ansible/sbin/build_image

@@ -2,6 +2,12 @@
 
 IMAGE_NAME=$1
 
+if [ -z "$1" ]; then
+    ONLY_IF_DATASET_MODIF=false
+else
+    ONLY_IF_DATASET_MODIF=true
+fi
+
 if [ -z "$IMAGE_NAME" ]; then
     echo "PAS DE NOM DE MODULE"
     exit 1
@@ -14,11 +20,11 @@ RISOTTO_IMAGE_DIR="$RISOTTO_DIR/images"
 IMAGE_BASE_RISOTTO_BASE_DIR="$RISOTTO_IMAGE_DIR/image_bases"
 IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP="$RISOTTO_IMAGE_DIR/tmp/$IMAGE_NAME"
 IMAGE_NAME_RISOTTO_IMAGE_DIR="$RISOTTO_IMAGE_DIR/$IMAGE_NAME"
-IMAGE_DIR_RECIPIENT_IMAGE="/var/lib/risotto/images_files/$IMAGE_NAME"
+IMAGE_DIR_RECIPIENT_IMAGE="$RISOTTO_DIR/images_files/$IMAGE_NAME"
 
 
 rm -f /var/log/risotto/build_image.log
-mkdir -p "$RISOTTO_IMAGE_DIR" "$RISOTTO_IMAGE_DIR/tmp/" /var/log/risotto
+mkdir -p "$RISOTTO_IMAGE_DIR" "$RISOTTO_IMAGE_DIR/tmp/"
 PKG=""
 BASE_DIR=""
 for script in $(ls "$IMAGE_DIR_RECIPIENT_IMAGE"/preinstall/*.sh 2> /dev/null); do
@@ -93,7 +99,7 @@ function install_pkg() {
 if [ ! -f "$BASE_LOCK" ] || [ ! -d "$BASE_DIR" ]; then
     echo "  - reinstallation de l'image de base"
     new_package_base
-    diff -u "$BASE_PKGS_FILE" "$BASE_PKGS_FILE".new && NEW_BASE=false || NEW_BASE=true
+    diff -u "$BASE_PKGS_FILE" "$BASE_PKGS_FILE".new &> /dev/null && NEW_BASE=false || NEW_BASE=true
     if [ ! -d "$BASE_DIR" ] || [ "$NEW_BASE" = true ]; then
         mkdir -p "$IMAGE_BASE_RISOTTO_BASE_DIR"
         rm -rf "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP"
@@ -121,7 +127,6 @@ if [ "$FUSION" = true ]; then
     dnf -y install "https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$RELEASEVER.noarch.rpm" --installroot="$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP" >> /var/log/risotto/build_image.log
 fi
 
-# FIXME verifier s'il y a des modifs sur pre/post
 if [ -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".base.pkgs ] && [ -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs ]; then
     echo "  - différence(s) avec les paquets de base"
     diff -u "$IMAGE_NAME_RISOTTO_IMAGE_DIR".base.pkgs "$BASE_PKGS_FILE" && INSTALL=false || INSTALL=true
@@ -130,13 +135,18 @@ else
     INSTALL=true
 fi
 
-new_package
+if [ "$ONLY_IF_DATASET_MODIF" = false ] || [ ! -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs ]; then
+    new_package
+else
+    cp --reflink=auto "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs.new
+fi
 if [ "$INSTALL" = false ]; then
     echo "  - différence(s) avec les paquets de l'image"
     diff -u "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs.new && INSTALL=false || INSTALL=true
 fi
 find "$IMAGE_DIR_RECIPIENT_IMAGE" -type f -exec md5sum '{}' \; > "$IMAGE_NAME_RISOTTO_IMAGE_DIR".md5sum.new
 if [ "$INSTALL" = false ]; then
+    echo "  - différence(s) du dataset"
     diff -u "$IMAGE_NAME_RISOTTO_IMAGE_DIR".md5sum "$IMAGE_NAME_RISOTTO_IMAGE_DIR".md5sum.new && INSTALL=false || INSTALL=true
 fi
 if [ "$INSTALL" = true ]; then
@@ -146,7 +156,11 @@ if [ "$INSTALL" = true ]; then
     else
         VERSION=0
     fi
+    if [ -d "$IMAGE_NAME_RISOTTO_IMAGE_DIR" ]; then
+        cd "$IMAGE_NAME_RISOTTO_IMAGE_DIR"
         make_changelog "$IMAGE_NAME" "$VERSION" "$OS_NAME" "$RELEASEVER" > "$IMAGE_NAME_RISOTTO_IMAGE_DIR"_"$RELEASEVER"_"$VERSION"_changelog.md
+        cd - > /dev/null
+    fi
     install_pkg
     sleep 2
     

ansible/sbin/compare_image

@@ -13,10 +13,24 @@ if [ ! -d "$dirname" ]; then
     exit 1
 fi
 cd $dirname
-find -type f | while read a; do
+find -type f -not -path "./secrets/*" -not -path "./tmpfiles.d/*" -not -path "./sysusers.d/*" -not -path "./systemd/*" -not -path "./tests/*" -not -path "./etc/pki/*" | while read a; do
-    cfile="/var/lib/machines/$SRV/usr/share/factory/$a"
+    machine_path="/var/lib/machines/$SRV"
+    cfile="$machine_path/usr/share/factory/$a"
     if [ -f "$cfile" ]; then
-        diff -u "$cfile" "$a"
+        diff -u "$dirname/$a" "$cfile"
+    else
+        FIRST_LINE="$(head -n 1 $a)"
+        if [[ "$FIRST_LINE" == "#RISOTTO: file://"* ]]; then
+            other=${FIRST_LINE:16}
+            diff -u "$dirname/$a" "$machine_path$other"
+        elif [[ "$FIRST_LINE" == "#RISOTTO: https://"* ]]; then
+            other=${FIRST_LINE:10}
+            echo $other
+            wget -q $other -O /tmp/template.tmp
+            diff -u "$dirname/$a" /tmp/template.tmp
+        elif [ ! "$FIRST_LINE" = "#RISOTTO: do not compare" ]; then
+            echo "cannot find \"$cfile\" ($dirname/$a)"
+        fi
     fi
 done
 cd - > /dev/null

ansible/sbin/make_changelog

@@ -90,7 +90,7 @@ def print_changelogs_markdown(packages):
             print(format_changelog_markdown(chl))
 
 
-def dnf_update(image_name):
+def dnf_update(image_name, releasever):
     conf = Conf()
     # obsoletes are already listed
     conf.obsoletes = False
@@ -102,7 +102,7 @@ def dnf_update(image_name):
         base.output = custom_output
         cli = Cli(base)
         image_dir = join(getcwd(), image_name)
-        cli.configure(['--setopt=install_weak_deps=False', '--nodocs', '--noplugins', '--installroot=' + image_dir, '--releasever', '35', 'check-update', '--changelog'], OptionParser())
+        cli.configure(['--setopt=install_weak_deps=False', '--nodocs', '--noplugins', '--installroot=' + image_dir, '--releasever', releasever, 'check-update', '--changelog'], OptionParser())
         logger = logging.getLogger("dnf")
         for h in logger.handlers:
             logger.removeHandler(h)
@@ -146,7 +146,7 @@ type = "installe"
         list_packages('Les paquets ajoutés', new_pkg - ori_pkg, new_dict)
         print('# Les paquets mises à jour\n')
         if os_name == 'fedora':
-            dnf_update(image_name)
+            dnf_update(image_name, releasever)
         else:
             for filename in glob('*.deb'):
                 unlink(filename)

ansible/sbin/make_volatile

@@ -1,5 +1,5 @@
 #!/bin/bash -e
-if [ -z $ROOT]; then
+if [ -z $ROOT ]; then
     echo "PAS DE ROOT"
     exit 1
 fi

ansible/sbin/test_images

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+QUIT_ON_ERROR=true
+# QUIT_ON_ERROR=false
+CONFIG_DIR="/var/lib/risotto/configurations"
+INFO_DIR="/var/lib/risotto/machines_informations"
+TEST_DIR="/var/lib/risotto/tests"
+TEST_DIR_NAME="tests"
+
+if [ ! -d /var/lib/risotto/tests/ ]; then
+    echo "no tests directory"
+    exit 1
+fi
+
+py_test_option="-s"
+if [ "$QUIT_ON_ERROR" = true ]; then
+    set -e
+    py_test_option="$py_test_option -x"
+fi
+
+for nspawn in $(ls /etc/systemd/nspawn/*.nspawn); do
+    nspawn_file=$(basename $nspawn)
+    machine=${nspawn_file%.*}
+    image=$(cat $INFO_DIR/$machine.image)
+    imagedir=$TEST_DIR/$image
+    machine_test_dir=$CONFIG_DIR/$machine/$TEST_DIR_NAME
+    export MACHINE_TEST_DIR=$machine_test_dir
+    echo "- $machine"
+    py.test-3 $py_test_option "$imagedir"
+done

ansible/sbin/update_images

@@ -6,18 +6,19 @@ RISOTTO_IMAGE_DIR="$RISOTTO_DIR/images"
 # image configuration
 IMAGE_BASE_RISOTTO_BASE_DIR="$RISOTTO_IMAGE_DIR/image_bases"
 
-rm -f $IMAGE_BASE_RISOTTO_BASE_DIR*.build
-
 if [ -z "$1" ]; then
-    ls /var/lib/risotto/images_files/ | while read image; do
+    rm -f $IMAGE_BASE_RISOTTO_BASE_DIR*.build
+fi
+
+mkdir -p /var/log/risotto
+
+ls /var/lib/risotto/images_files/ | while read image; do
     if [ -d /var/lib/risotto/images_files/"$image" ]; then
         echo
-            echo "Install image $image"
+        echo "Install image $image" | tee -a /var/log/risotto/update_images.log
-            /usr/local/sbin/build_image "$image" || true
+        /usr/local/sbin/build_image "$image" "$1" | tee -a /var/log/risotto/update_images.log || (echo "PROBLEME" | tee -a /var/log/risotto/update_images.log; true)
     fi
-    done
+done
-fi
-#rm -f $IMAGE_BASE_RISOTTO_BASE_DIR*.build
 
 MACHINES=""
 for nspawn in $(ls /etc/systemd/nspawn/*.nspawn); do
@@ -25,10 +26,10 @@ for nspawn in $(ls /etc/systemd/nspawn/*.nspawn); do
     machine=${nspawn_file%.*}
     MACHINES="$MACHINES$machine "
     MACHINE_MACHINES_DIR="/var/lib/machines/$machine"
-    SHA_MACHINE="$RISOTTO_DIR/configurations/sha/$machine".sha
+    IMAGE_NAME_RISOTTO_IMAGE_NAME="$(cat $RISOTTO_DIR/machines_informations/$machine.image)"
-    content=$(cat $SHA_MACHINE)
+    MACHINE_INFO="$RISOTTO_DIR/machines_informations/"
-    IMAGE_NAME_RISOTTO_IMAGE_NAME=${content##* }
+    VERSION_MACHINE="$MACHINE_INFO/$machine.version"
-    diff -q "$IMAGE_NAME_RISOTTO_IMAGE_NAME".sha "$SHA_MACHINE" > /dev/null || (
+    diff -q "$RISOTTO_IMAGE_DIR/$IMAGE_NAME_RISOTTO_IMAGE_NAME".version "$VERSION_MACHINE" &> /dev/null || (
         echo "Reinstall machine $machine"
         machinectl stop $machine || true
         while true; do
@@ -37,10 +38,12 @@ for nspawn in $(ls /etc/systemd/nspawn/*.nspawn); do
         done
         rm -rf "$MACHINE_MACHINES_DIR"
         mkdir "$MACHINE_MACHINES_DIR"
-        cd "$MACHINE_MACHINES_DIR"
+        cp -a --reflink=auto $RISOTTO_IMAGE_DIR/$IMAGE_NAME_RISOTTO_IMAGE_NAME/* $MACHINE_MACHINES_DIR
-        tar xf "$IMAGE_NAME_RISOTTO_IMAGE_NAME"
+        cp -a --reflink=auto "$RISOTTO_IMAGE_DIR/$IMAGE_NAME_RISOTTO_IMAGE_NAME".version "$VERSION_MACHINE"
-        cp -a "$IMAGE_NAME_RISOTTO_IMAGE_NAME".sha "$SHA_MACHINE"
     )
 done
-machinectl start $MACHINES
+if [ -z "$1" ]; then
-diagnose
+    machinectl start $MACHINES
+    diagnose
+fi
+exit 0

doc/authentification.svg

@@ -0,0 +1,405 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   width="210mm"
+   height="297mm"
+   viewBox="0 0 210 297"
+   version="1.1"
+   id="svg5"
+   inkscape:version="1.2.1 (9c6d41e410, 2022-07-14)"
+   sodipodi:docname="authentification.svg"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <sodipodi:namedview
+     id="namedview7"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:document-units="mm"
+     showgrid="false"
+     inkscape:zoom="1.557211"
+     inkscape:cx="355.12207"
+     inkscape:cy="181.73517"
+     inkscape:window-width="2048"
+     inkscape:window-height="1083"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="layer1"
+     inkscape:showpageshadow="2"
+     inkscape:deskcolor="#d1d1d1" />
+  <defs
+     id="defs2">
+    <rect
+       x="465.27745"
+       y="390.53444"
+       width="155.19784"
+       height="121.34324"
+       id="rect9339" />
+    <marker
+       style="overflow:visible"
+       id="marker47493"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Sstart"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(0.2,0,0,0.2,1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path47491" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Arrow2Send"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow2Send"
+       inkscape:isstock="true"
+       viewBox="0 0 3.4652294 2.5981128"
+       markerWidth="3.465229"
+       markerHeight="2.5981126"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="matrix(-0.3,0,0,-0.3,0.69,0)"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:0.625;stroke-linejoin:round"
+         id="path45715" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Arrow2Sstart"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow2Sstart"
+       inkscape:isstock="true"
+       viewBox="0 0 3.4652294 2.5981128"
+       markerWidth="3.465229"
+       markerHeight="2.5981126"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="matrix(0.3,0,0,0.3,-0.69,0)"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:0.625;stroke-linejoin:round"
+         id="path45712" />
+    </marker>
+    <marker
+       style="overflow:visible;"
+       id="Arrow1Send"
+       refX="0.0"
+       refY="0.0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true">
+      <path
+         transform="scale(0.2) rotate(180) translate(6,0)"
+         style="fill-rule:evenodd;fill:context-stroke;stroke:context-stroke;stroke-width:1.0pt;"
+         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
+         id="path45697" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Arrow1Sstart"
+       refX="0.0"
+       refY="0.0"
+       orient="auto"
+       inkscape:stockid="Arrow1Sstart"
+       inkscape:isstock="true">
+      <path
+         transform="scale(0.2) translate(6,0)"
+         style="fill-rule:evenodd;fill:context-stroke;stroke:context-stroke;stroke-width:1.0pt"
+         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
+         id="path45694" />
+    </marker>
+    <marker
+       style="overflow:visible;"
+       id="Arrow1Lend"
+       refX="0.0"
+       refY="0.0"
+       orient="auto"
+       inkscape:stockid="Arrow1Lend"
+       inkscape:isstock="true">
+      <path
+         transform="scale(0.8) rotate(180) translate(12.5,0)"
+         style="fill-rule:evenodd;fill:context-stroke;stroke:context-stroke;stroke-width:1.0pt;"
+         d="M 0.0,0.0 L 5.0,-5.0 L -12.5,0.0 L 5.0,5.0 L 0.0,0.0 z "
+         id="path45685" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179-3"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true"
+       viewBox="0 0 4.4434635 2.539122"
+       markerWidth="4.4434633"
+       markerHeight="2.5391221"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177-6" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179-3-6"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177-6-2" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179-1"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true"
+       viewBox="0 0 4.4434635 2.539122"
+       markerWidth="4.4434633"
+       markerHeight="2.5391221"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177-8" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179-9"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177-2" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179-9-3"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177-2-7" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179-3-6-9"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177-6-2-2" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Arrow2Sstart-8"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow2Sstart"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(0.3,0,0,0.3,-0.69,0)"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:0.625;stroke-linejoin:round"
+         id="path45712-9" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179-3-6-9-7"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177-6-2-2-3" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179-3-6-4"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true"
+       viewBox="0 0 4.4434635 2.539122"
+       markerWidth="4.4434633"
+       markerHeight="2.5391221"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177-6-2-7" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Arrow2Send-3"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow2Send"
+       inkscape:isstock="true"
+       viewBox="0 0 3.4652294 2.5981128"
+       markerWidth="3.465229"
+       markerHeight="2.5981126"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="matrix(-0.3,0,0,-0.3,0.69,0)"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:0.625;stroke-linejoin:round"
+         id="path45715-6" />
+    </marker>
+  </defs>
+  <g
+     inkscape:label="Calque 1"
+     inkscape:groupmode="layer"
+     id="layer1">
+    <ellipse
+       style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:5.86795;stroke-linecap:square;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;paint-order:fill markers stroke"
+       id="path986"
+       cx="62.406502"
+       cy="64.119804"
+       rx="4.5660253"
+       ry="4.5660257" />
+    <ellipse
+       style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:5.86795;stroke-linecap:square;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;paint-order:fill markers stroke"
+       id="path986-6-5"
+       cx="142.10091"
+       cy="64.120003"
+       rx="4.5660253"
+       ry="4.5660257" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.93889px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="63.8083"
+       y="77.278412"
+       id="text3135"><tspan
+         sodipodi:role="line"
+         style="font-weight:bold;font-size:4.93889px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+         x="63.8083"
+         y="77.278412"
+         id="tspan10911">IMAP</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-size:4.93889px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="135.16226"
+       y="78.779442"
+       id="text3135-7-6"><tspan
+         sodipodi:role="line"
+         style="font-weight:bold;font-size:4.93889px;stroke-width:0.264583"
+         x="135.16226"
+         y="78.779442"
+         id="tspan10911-3-2">LDAP</tspan></text>
+    <path
+       style="fill:none;stroke:#000000;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Send)"
+       d="M 75.106998,57.935664 H 122.14408"
+       id="path46175-02"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Send-3)"
+       d="M 75.340285,68.861719 H 122.37734"
+       id="path46175-02-7"
+       sodipodi:nodetypes="cc" />
+    <path
+       inkscape:connector-curvature="0"
+       style="opacity:0.98;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.175296;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 94.078762,40.916652 c -0.0389,2.57e-4 -0.0774,5.85e-4 -0.11611,0.0022 -1.77767,0.0113 -3.37259,1.592182 -3.37712,3.374607 -0.0202,0.420172 0.007,0.840654 0,1.260955 -0.28547,-7e-6 -0.57094,-7e-6 -0.85649,0 0.023,1.846787 -0.0461,3.697844 0.036,5.54194 0.17721,1.1875 1.24351,2.26136 2.48695,2.203553 1.36149,-0.0022 2.72716,0.04211 4.086269,-0.0275 1.275754,-0.219817 2.171678,-1.529827 2.074938,-2.797815 0.0144,-1.639617 0,-3.279313 0.007,-4.918966 -0.284237,-0.0072 -0.568484,0.005 -0.852724,-0.0036 0.0216,-0.998381 0.0684,-2.089696 -0.500617,-2.955111 -0.615417,-1.026965 -1.788466,-1.688137 -2.987566,-1.680443 z m 0.0165,1.425752 c 1.01001,0.01389 2.00786,0.850284 1.97878,1.902665 0.0202,0.436339 0.0331,0.872937 0.0425,1.309642 -1.35875,-5.85e-4 -2.71751,0.0022 -4.07619,-0.0022 0.007,-0.683077 -0.17908,-1.429948 0.19471,-2.044983 0.33945,-0.651636 1.01793,-1.150287 1.76284,-1.163575 0.0324,-0.0015 0.0648,-0.0022 0.0974,-0.0015 z"
+       id="path3355" />
+    <rect
+       style="fill:none;fill-rule:evenodd;stroke:#040000;stroke-width:1.27229;stroke-linecap:square;stroke-dasharray:none;stroke-opacity:1;paint-order:fill markers stroke"
+       id="rect8947"
+       width="29.594231"
+       height="6.274775"
+       x="79.703773"
+       y="82.478172" />
+    <text
+       xml:space="preserve"
+       style="font-size:11.9191px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.15963"
+       x="80.080597"
+       y="91.894714"
+       id="text9263"><tspan
+         sodipodi:role="line"
+         id="tspan9261"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:11.9191px;font-family:'Abyssinica SIL';-inkscape-font-specification:'Abyssinica SIL Bold';stroke-width:0.15963"
+         x="80.080597"
+         y="91.894714">*****</tspan></text>
+    <rect
+       style="fill:none;fill-rule:evenodd;stroke:#040000;stroke-width:1.27229;stroke-linecap:square;stroke-dasharray:none;stroke-opacity:1;paint-order:fill markers stroke"
+       id="rect8947-5"
+       width="29.594231"
+       height="6.274775"
+       x="79.942833"
+       y="74.1101" />
+    <text
+       xml:space="preserve"
+       style="font-size:7.05556px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.15963"
+       x="80.848824"
+       y="79.293304"
+       id="text9263-6"><tspan
+         sodipodi:role="line"
+         id="tspan9261-2"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.05556px;font-family:'Abyssinica SIL';-inkscape-font-specification:'Abyssinica SIL Bold';stroke-width:0.15963"
+         x="80.848824"
+         y="79.293304">domaine</tspan></text>
+    <text
+       xml:space="preserve"
+       transform="scale(0.26458333)"
+       id="text9337"
+       style="font-size:40px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;white-space:pre;shape-inside:url(#rect9339);display:inline" />
+  </g>
+</svg>

doc/example_smtp.svg

@@ -0,0 +1,466 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   width="178.66008mm"
+   height="172.7524mm"
+   viewBox="0 0 178.66008 172.7524"
+   version="1.1"
+   id="svg5"
+   inkscape:version="1.2.1 (9c6d41e410, 2022-07-14)"
+   sodipodi:docname="example_smtp.svg"
+   inkscape:export-filename="example_smtp.png"
+   inkscape:export-xdpi="149.26"
+   inkscape:export-ydpi="149.26"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <sodipodi:namedview
+     id="namedview7"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:document-units="mm"
+     showgrid="false"
+     inkscape:zoom="0.55055723"
+     inkscape:cx="173.46062"
+     inkscape:cy="599.39273"
+     inkscape:window-width="1920"
+     inkscape:window-height="1011"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="layer1"
+     inkscape:showpageshadow="2"
+     inkscape:deskcolor="#d1d1d1" />
+  <defs
+     id="defs2">
+    <marker
+       style="overflow:visible"
+       id="marker47493"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Sstart"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(0.2,0,0,0.2,1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path47491" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Arrow2Send"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow2Send"
+       inkscape:isstock="true"
+       viewBox="0 0 3.4652294 2.5981128"
+       markerWidth="3.465229"
+       markerHeight="2.5981126"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="matrix(-0.3,0,0,-0.3,0.69,0)"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:0.625;stroke-linejoin:round"
+         id="path45715" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Arrow2Sstart"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow2Sstart"
+       inkscape:isstock="true"
+       viewBox="0 0 3.4652294 2.5981128"
+       markerWidth="3.465229"
+       markerHeight="2.5981126"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="matrix(0.3,0,0,0.3,-0.69,0)"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:0.625;stroke-linejoin:round"
+         id="path45712" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Arrow1Send"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path45697" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Arrow1Sstart"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Sstart"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(0.2,0,0,0.2,1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path45694" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Arrow1Lend"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Lend"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.8,0,0,-0.8,-10,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path45685" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179-3"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true"
+       viewBox="0 0 4.4434635 2.539122"
+       markerWidth="4.4434633"
+       markerHeight="2.5391221"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177-6" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179-3-6"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177-6-2" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179-1"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true"
+       viewBox="0 0 4.4434635 2.539122"
+       markerWidth="4.4434633"
+       markerHeight="2.5391221"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177-8" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179-9"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177-2" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179-9-3"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177-2-7" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179-3-6-9"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177-6-2-2" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Arrow2Sstart-8"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow2Sstart"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(0.3,0,0,0.3,-0.69,0)"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:0.625;stroke-linejoin:round"
+         id="path45712-9" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179-3-6-9-7"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177-6-2-2-3" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker46179-3-6-4"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Arrow1Send"
+       inkscape:isstock="true"
+       viewBox="0 0 4.4434635 2.539122"
+       markerWidth="4.4434633"
+       markerHeight="2.5391221"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         id="path46177-6-2-7" />
+    </marker>
+  </defs>
+  <g
+     inkscape:label="Calque 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(-15.292364,-14.109702)">
+    <rect
+       style="fill:#ffffff;fill-opacity:1;fill-rule:evenodd;stroke:#f6f7d7;stroke-width:0.600001;stroke-linecap:round;stroke-linejoin:round;paint-order:fill markers stroke"
+       id="rect443"
+       width="178.06007"
+       height="172.15239"
+       x="15.592364"
+       y="14.409702" />
+    <circle
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:4;stroke-linecap:square;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;paint-order:fill markers stroke"
+       id="path846"
+       cx="96.73632"
+       cy="103.80212"
+       r="52.962326" />
+    <ellipse
+       style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:5.86795;stroke-linecap:square;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;paint-order:fill markers stroke"
+       id="path986"
+       cx="62.406502"
+       cy="64.119804"
+       rx="4.5660253"
+       ry="4.5660257" />
+    <ellipse
+       style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:5.86795;stroke-linecap:square;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;paint-order:fill markers stroke"
+       id="path986-6"
+       cx="62.407001"
+       cy="144.45392"
+       rx="4.5660253"
+       ry="4.5660257" />
+    <ellipse
+       style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:5.86795;stroke-linecap:square;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;paint-order:fill markers stroke"
+       id="path986-6-5"
+       cx="98.457001"
+       cy="79.992493"
+       rx="4.5660253"
+       ry="4.5660257" />
+    <ellipse
+       style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:5.86795;stroke-linecap:square;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;paint-order:fill markers stroke"
+       id="path986-6-5-7"
+       cx="98.45739"
+       cy="122.40948"
+       rx="4.5660253"
+       ry="4.5660257" />
+    <ellipse
+       style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:5.86795;stroke-linecap:square;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;paint-order:fill markers stroke"
+       id="path986-6-5-9"
+       cx="149.40425"
+       cy="102.7455"
+       rx="4.5660253"
+       ry="4.5660257" />
+    <text
+       xml:space="preserve"
+       style="font-size:4.93889px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="37.39616"
+       y="61.122501"
+       id="text3135"><tspan
+         sodipodi:role="line"
+         id="tspan3133"
+         style="font-weight:bold;font-size:4.93889px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+         x="37.39616"
+         y="61.122501">IMAP (993)</tspan><tspan
+         sodipodi:role="line"
+         style="font-weight:bold;font-size:4.93889px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+         x="37.39616"
+         y="67.296112"
+         id="tspan10911">SMTP (587)</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-size:4.93889px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="62.512436"
+       y="157.8011"
+       id="text3135-7"><tspan
+         sodipodi:role="line"
+         style="font-weight:bold;font-size:4.93889px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+         x="62.512436"
+         y="157.8011"
+         id="tspan10911-3">SMTP</tspan><tspan
+         sodipodi:role="line"
+         style="font-weight:bold;font-size:4.93889px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+         x="62.512436"
+         y="163.97472"
+         id="tspan51838">relay</tspan><tspan
+         sodipodi:role="line"
+         style="font-weight:bold;font-size:4.93889px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+         x="62.512436"
+         y="170.14833"
+         id="tspan51842">(25)</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-size:4.93889px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;stroke-width:0.264583"
+       x="91.762589"
+       y="70.189774"
+       id="text3135-7-6"><tspan
+         sodipodi:role="line"
+         style="font-weight:bold;font-size:4.93889px;stroke-width:0.264583"
+         x="91.762589"
+         y="70.189774"
+         id="tspan10911-3-2">LDAP</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-size:4.93889px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;stroke-width:0.264583"
+       x="162.68114"
+       y="114.31043"
+       id="text3135-7-6-1"><tspan
+         sodipodi:role="line"
+         style="font-weight:bold;font-size:4.93889px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+         x="162.68114"
+         y="114.31043"
+         id="tspan10911-3-2-2">DNS</tspan><tspan
+         sodipodi:role="line"
+         style="font-weight:bold;font-size:4.93889px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+         x="162.68114"
+         y="120.48405"
+         id="tspan21295">Résolver</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-size:4.93889px;line-height:1.25;font-family:sans-serif;text-align:center;letter-spacing:0px;word-spacing:0px;text-anchor:middle;stroke-width:0.264583"
+       x="98.739502"
+       y="135.19682"
+       id="text3135-7-6-1-0"><tspan
+         sodipodi:role="line"
+         style="font-weight:bold;font-size:4.93889px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+         x="98.739502"
+         y="135.19682"
+         id="tspan10911-3-2-2-9">DNS</tspan><tspan
+         sodipodi:role="line"
+         style="font-weight:bold;font-size:4.93889px;text-align:center;text-anchor:middle;stroke-width:0.264583"
+         x="98.739502"
+         y="141.37044"
+         id="tspan22983">autoritaire</tspan></text>
+    <path
+       style="fill:none;stroke:#000000;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Send)"
+       d="M 98.411,89.863152 V 107.09425"
+       id="path46175"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Send)"
+       d="m 69.168475,71.05256 15.523919,4.588191"
+       id="path46175-02"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#Arrow2Sstart);marker-end:url(#Arrow2Send)"
+       d="m 58.918881,78.428313 c 0,0 -7.642846,11.083665 -8.1137,23.703427 -0.554549,14.86295 7.598141,26.95783 7.598141,26.95783"
+       id="path46175-02-5"
+       sodipodi:nodetypes="csc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Send)"
+       d="m 73.470622,64.314043 c 0,0 23.019562,-13.687982 46.104948,-0.359501 9.42693,5.44269 13.02345,12.067909 16.41683,17.107652 3.97188,5.898933 4.72416,9.274399 4.72416,9.274399"
+       id="path46175-7"
+       sodipodi:nodetypes="cssc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Send)"
+       d="m 140.00042,103.64348 -27.82831,13.38506"
+       id="path46175-0"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Send)"
+       d="m 158.873,103.231 19.41573,1.3e-4"
+       id="path46175-0-6"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Send)"
+       d="M 38.595,36.174542 51.766,51.796621"
+       id="path46175-0-6-8"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#Arrow2Sstart);marker-end:url(#Arrow2Send)"
+       d="M 51.766,154.77542 38.595,168.15219"
+       id="path46175-0-6-2-6"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:4;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Send)"
+       d="m 73.436962,143.41602 c 0,0 17.397816,13.36128 44.232048,1.48383 16.06906,-7.11254 23.91983,-29.57648 23.91983,-29.57648"
+       id="path57407"
+       sodipodi:nodetypes="csc" />
+  </g>
+</svg>

sbin/risotto_auto_doc

@@ -77,6 +77,8 @@ def parse(applicationservice, elts, dico, providers_suppliers, hidden):
                         values['type'] = child.type
                     if hasattr(child, 'default'):
                         default = child.default
+                        if isinstance(default, objectspace.value):
+                            default = '<calculated>'
                         if isinstance(default, list):
                             default = '<br />'.join(default)
                         values['values'] = default
@@ -151,8 +153,8 @@ for applicationservice, applicationservice_data in applicationservices_data.item
     if not isdir(dirname) and not extra_dictionaries:
         continue
     rougailconfig['extra_dictionaries'] = extra_dictionaries
-    converted = RougailConvert(rougailconfig)
+    converted = RougailConvert(rougailconfig, just_doc=True)
-    converted.load_dictionaries(just_doc=True)
+    converted.load_dictionaries()
     converted.annotate()
     objectspace = converted.rougailobjspace
     if hasattr(objectspace.space, 'variables'):
@@ -231,6 +233,10 @@ for applicationservice, applicationservice_data in applicationservices_data.item
             as_fh.write('\n- [+]: variable is multiple\n- **bold**: variable is mandatory\n')
         if applicationservice_data['used_by']:
             as_fh.write('\n## Used by\n\n')
+            if len(applicationservice_data['used_by']) == 1:
+                link = applicationservice_data['used_by'][0]
+                as_fh.write(f'[{link}](../{link}/README.md)\n')
+            else:
                 for link in applicationservice_data['used_by']:
                     as_fh.write(f'- [{link}](../{link}/README.md)\n')
         linked = []
@@ -238,22 +244,35 @@ for applicationservice, applicationservice_data in applicationservices_data.item
             if not applicationservice in provider_as:
                 continue
             for supplier in providers_suppliers['suppliers'][provider]:
-                if not linked:
-                    as_fh.write('\n## Linked to\n\n')
                 if supplier in linked:
                     continue
-                as_fh.write(f'- [{supplier}](../{supplier}/README.md)\n')
                 linked.append(supplier)
+        linked.sort()
+        if linked:
+            if len(linked) == 1:
+                as_fh.write('\n## Supplier\n\n')
+                as_fh.write(f'[{linked[0]}](../{linked[0]}/README.md)\n')
+            else:
+                as_fh.write('\n## Suppliers\n\n')
+                for supplier in linked:
+                    as_fh.write(f'- [{supplier}](../{supplier}/README.md)\n')
+        linked = []
         for supplier, supplier_as in providers_suppliers['suppliers'].items():
             if not applicationservice in supplier_as:
                 continue
             for provider in providers_suppliers['providers'][supplier]:
-                if not linked:
-                    as_fh.write('\n## Linked to\n\n')
                 if provider in linked:
                     continue
-                as_fh.write(f'- [{provider}](../{provider}/README.md)\n')
                 linked.append(provider)
+        linked.sort()
+        if linked:
+            if len(linked) == 1:
+                as_fh.write('\n## Provider\n\n')
+                as_fh.write(f'[{linked[0]}](../{linked[0]}/README.md)\n')
+            else:
+                as_fh.write('\n## Providers\n\n')
+                for provider in linked:
+                    as_fh.write(f'- [{provider}](../{provider}/README.md)\n')
 
 
 with open('seed/README.md', 'w') as as_fh:
@@ -275,3 +294,24 @@ with open('seed/README.md', 'w') as as_fh:
             for applicationservice in applicationservices_:
                 applicationservice_data = applicationservices_data[applicationservice]
                 as_fh.write(f'  - [{applicationservice}]({applicationservice}/README.md): {applicationservice_data["description"]}\n')
+    as_fh.write('\n# Providers and suppliers\n\n')
+    providers = list(providers_suppliers['providers'].keys())
+    providers.sort()
+    for provider in providers:
+        as_fh.write(f'- {provider}:\n')
+        if providers_suppliers['providers'][provider]:
+            if len(providers_suppliers['providers'][provider]) == 1:
+                applicationservice = providers_suppliers['providers'][provider][0]
+                as_fh.write(f'  - Provider: [{applicationservice}]({applicationservice}/README.md)\n')
+            else:
+                as_fh.write(f'  - Providers:\n')
+                for applicationservice in providers_suppliers['providers'][provider]:
+                    as_fh.write(f'    - [{applicationservice}]({applicationservice}/README.md)\n')
+        if providers_suppliers['suppliers']:
+            if len(providers_suppliers['suppliers'][provider]) == 1:
+                applicationservice = providers_suppliers['suppliers'][provider][0]
+                as_fh.write(f'  - Supplier: [{applicationservice}]({applicationservice}/README.md)\n')
+            else:
+                as_fh.write(f'  - Suppliers:\n')
+                for applicationservice in providers_suppliers['suppliers'][provider]:
+                    as_fh.write(f'    - [{applicationservice}]({applicationservice}/README.md)\n')

sbin/risotto_templates

@@ -12,14 +12,19 @@ async def main():
     parser.add_argument('server_name')
     parser.add_argument('--nocache', action='store_true')
     parser.add_argument('--debug', action='store_true')
+    parser.add_argument('--copy_tests', action='store_true')
+    parser.add_argument('--template')
     args = parser.parse_args()
     if args.nocache:
         remove_cache()
 
-    config = await load()
+    config = await load(copy_tests=args.copy_tests, clean_directories=True)
+    print('fin')
+    print(await config.option('host_example_net.general.copy_tests').value.get())
     try:
         await templates(args.server_name,
                         config,
+                        template=args.template
                         )
     except Exception as err:
         if args.debug:

src/risotto/image.py

@@ -2,6 +2,7 @@ from shutil import copy2, copytree
 from os import listdir, makedirs
 from os.path import join, isdir, isfile, dirname
 from yaml import load as yaml_load, SafeLoader
+from tiramisu.error import PropertiesOptionError
 #
 from .utils import RISOTTO_CONFIG
 
@@ -181,11 +182,9 @@ class Modules:
                 if isdir(extra_dir):
                     cfg.extra_dictionaries.setdefault(extra, []).append(extra_dir)
         # manual
-        for type in ['image', 'install']:
+        manual_dir = join(as_dir, 'manual', 'image')
-            manual_dir = join(as_dir, 'manual')
+        if isdir(manual_dir):
-            if isdir(join(manual_dir, type)):
             cfg.manuals.append(manual_dir)
-                break
         # tests
         tests_dir = join(as_dir, 'tests')
         if isdir(tests_dir):
@@ -218,6 +217,10 @@ def applicationservice_copy(src_file: str,
 
 async def valid_mandatories(config):
     mandatories = await config.value.mandatory()
+    await config.property.pop('mandatory')
+    hidden = {}
+    variables = {}
+    title = None
     if mandatories:
         server_name = None
         for mandatory in mandatories:
@@ -225,10 +228,20 @@ async def valid_mandatories(config):
             var_server_name = await config.option(path_server_name).option.description()
             if server_name != var_server_name:
                 server_name = var_server_name
-                print()
+                title = f'=== Missing variables for {server_name} ==='
-                print(f'=== Missing variables for {server_name} ===')
+            suboption = config.option(mandatory)
-            print(f' - {path}')
+            text = await suboption.option.doc()
-            # await config.property.pop('mandatory')
+            msg = f' - {text} ({path})'
-            # await value_pprint(await config.value.dict(), config)
+            supplier = await suboption.information.get('supplier', None)
-        exit(1)
+            if supplier:
-        #raise Exception('configuration has mandatories variables without values')
+                msg += f' you could add a service that provides {supplier}'
+            try:
+                await config.option(mandatory).value.get()
+                variables.setdefault(title, []).append(msg)
+            except PropertiesOptionError as err:
+                if 'hidden' not in err.proptype:
+                    raise PropertiesOptionError(err)
+                hidden.setdefault(title, []).append(msg)
+        if not variables:
+            variables = hidden
+        return variables

src/risotto/machine.py

@@ -3,15 +3,15 @@ from .image import Applications, Modules, valid_mandatories, applicationservice_
 from .rougail.annotator import calc_providers, calc_providers_global, calc_providers_dynamic, calc_providers_dynamic_follower, calc_providers_follower
 
 from rougail import RougailConfig, RougailConvert
-from os import remove, makedirs, listdir
+from os import remove, makedirs, listdir, chmod
-from os.path import isfile, isdir, abspath
+from os.path import isfile, isdir, abspath, join, dirname
 from json import dump as json_dump, load as json_load
 from yaml import load as yaml_load, SafeLoader
 #
 from tiramisu import Config, valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal, calc_value
 from rougail.utils import normalize_family
 from rougail import RougailSystemdTemplate
-from shutil import rmtree
+from shutil import copy2, copytree, rmtree
 
 
 def tiramisu_display_name(kls,
@@ -31,7 +31,10 @@ TIRAMISU_CACHE = 'tiramisu_cache.py'
 VALUES_CACHE = 'values_cache.json'
 INFORMATIONS_CACHE = 'informations_cache.json'
 INSTALL_DIR = RISOTTO_CONFIG['directories']['dest']
-INSTALL_TEMPLATES_DIR = RISOTTO_CONFIG['directories']['dest_templates']
+INSTALL_CONFIG_DIR = 'configurations'
+INSTALL_TMPL_DIR= 'templates'
+INSTALL_IMAGES_DIR = 'images_files'
+INSTALL_TESTS_DIR = 'tests'
 FUNCTIONS = {'calc_providers': calc_providers,
              'calc_providers_global': calc_providers_global,
              'calc_providers_dynamic': calc_providers_dynamic,
@@ -47,10 +50,32 @@ FUNCTIONS = {'calc_providers': calc_providers,
              }
 
 
-def re_create(dirname):
+def copy(src_file, dst_file):
-    if isdir(dirname):
+    if isdir(src_file):
-        rmtree(dirname)
+        if not isdir(dst_file):
-    makedirs(dirname)
+            makedirs(dst_file)
+        for subfilename in listdir(src_file):
+            if not isfile(dst_file):
+                src = join(src_file, subfilename)
+                dst = join(dst_file, subfilename)
+                if isfile(src):
+                    copy2(src, dst)
+                else:
+                    copytree(src, dst)
+    elif not isfile(dst_file):
+        dst = dirname(dst_file)
+        if not isdir(dst):
+            makedirs(dst)
+        if isfile(src_file):
+            copy2(src_file, dst_file)
+        else:
+            copytree(src_file, dst_file)
+
+
+def re_create(dir_name):
+    if isdir(dir_name):
+        rmtree(dir_name)
+    makedirs(dir_name)
 
 
 def remove_cache():
@@ -65,6 +90,8 @@ def remove_cache():
 async def templates(server_name,
                     config,
                     just_copy=False,
+                    copy_manuals=False,
+                    template=None,
                     ):
     subconfig = config.option(normalize_family(server_name))
     try:
@@ -77,23 +104,25 @@ async def templates(server_name,
     rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE
     rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
     rougailconfig['tmp_dir'] = 'tmp'
-    if not just_copy:
-        rougailconfig['destinations_dir'] = INSTALL_DIR
-    else:
-        rougailconfig['destinations_dir'] = INSTALL_TEMPLATES_DIR
     rougailconfig['templates_dir'] = await subconfig.information.get('templates_dir')
     rougailconfig['patches_dir'] = await subconfig.information.get('patches_dir')
     rougailconfig['functions_file'] = await subconfig.information.get('functions_files')
-    is_host = await subconfig.information.get('module') == 'host'
+    module = await subconfig.information.get('module')
+    is_host = module == 'host'
     if is_host:
-        host_install_dir = f'{ROUGAIL_NAMESPACE}.host_install_dir'
+        rougailconfig['systemd_tmpfile_delete_before_create'] = True
-        rougailconfig['tmpfile_dest_dir'] = await subconfig.option(host_install_dir).value.get()
+        if just_copy:
-        rougailconfig['default_systemd_directory'] = '/usr/local/lib/systemd'
+            raise Exception('cannot generate template with option just_copy for a host')
     else:
-        rougailconfig['tmpfile_dest_dir'] = '/usr/local/lib'
+        rougailconfig['systemd_tmpfile_delete_before_create'] = False
-        rougailconfig['default_systemd_directory'] = '/systemd'
+    #rougailconfig['systemd_tmpfile_factory_dir'] = '/usr/local/lib'
+    if not just_copy:
+        rougailconfig['destinations_dir'] = join(INSTALL_DIR, INSTALL_CONFIG_DIR, server_name)
+    else:
+        rougailconfig['destinations_dir'] = join(INSTALL_DIR, INSTALL_TMPL_DIR, server_name)
     re_create(rougailconfig['destinations_dir'])
     re_create(rougailconfig['tmp_dir'])
+
     engine = RougailSystemdTemplate(subconfig, rougailconfig)
     if just_copy:
         # for all engine to none
@@ -104,7 +133,10 @@ async def templates(server_name,
             ori_engines[eng] = engine.engines[eng]
             engine.engines[eng] = engine.engines['none']
     try:
+        if not template:
             await engine.instance_files()
+        else:
+            await engine.instance_file(template)
     except Exception as err:
         print()
         print(f'=== Configuration: {server_name} ===')
@@ -114,22 +146,37 @@ async def templates(server_name,
     if just_copy:
         for eng, old_engine in ori_engines.items():
             engine.engines[eng] = old_engine
+    secrets_dir = join(rougailconfig['destinations_dir'], 'secrets')
+    if isdir(secrets_dir):
+        chmod(secrets_dir, 0o700)
+    if copy_manuals and not is_host:
+        dest_dir = join(INSTALL_DIR, INSTALL_IMAGES_DIR, module)
+        if not isdir(dest_dir):
+            for manual in await subconfig.information.get('manuals_dirs'):
+                for filename in listdir(manual):
+                    src_file = join(manual, filename)
+                    dst_file =  join(dest_dir, filename)
+                    copy(src_file, dst_file)
+    copy_tests = await config.information.get('copy_tests')
+
+    if copy_tests and not is_host:
+        dest_dir = join(INSTALL_DIR, INSTALL_TESTS_DIR, module)
+        if not isdir(dest_dir):
+            for tests in await subconfig.information.get('tests_dirs'):
+                for filename in listdir(tests):
+                    src_file = join(tests, filename)
+                    dst_file =  join(dest_dir, filename)
+                    copy(src_file, dst_file)
 
 
 class Loader:
     def __init__(self,
-                 cache_file,
-                 cache_values,
-                 cache_informations,
                  clean_directories,
                  hide_secret,
                  original_display_name,
                  valid_mandatories,
                  config_file=CONFIG_FILE,
                  ):
-        self.cache_file = cache_file
-        self.cache_values = cache_values
-        self.cache_informations = cache_informations
         self.hide_secret = hide_secret
         self.original_display_name = original_display_name
         self.valid_mandatories = valid_mandatories
@@ -139,9 +186,12 @@ class Loader:
                 rmtree(INSTALL_DIR)
             makedirs(INSTALL_DIR)
 
-    def before(self):
+    def load_tiramisu_file(self):
+        """Load config file (servers.yml) and build tiramisu file with dataset informations
+        """
         with open(self.config_file, 'r') as server_fh:
             self.servers_json = yaml_load(server_fh, Loader=SafeLoader)
+        # set global rougail configuration
         cfg = RougailConfig.copy()
         cfg['variable_namespace'] = ROUGAIL_NAMESPACE
         cfg['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
@@ -151,29 +201,48 @@ class Loader:
         cfg['force_convert_dyn_option_description'] = True
         cfg['risotto_globals'] = {}
 
-        rougail = RougailConvert(cfg)
+        # initialise variables to store useful informations
+        # those variables are use during templating
         self.templates_dir = {}
         self.patches_dir = {}
-        functions_files = set()
         self.functions_files = {}
+        self.manuals_dirs = {}
+        self.tests_dirs = {}
+        self.modules = {}
+
+        functions_files = set()
         applicationservices = Applications()
         zones = self.servers_json['zones']
-        self.modules = {}
+
+        rougail = RougailConvert(cfg)
         for host_name, datas in self.servers_json['hosts'].items():
-            modules_name = {mod_datas['module'] for mod_datas in datas['servers'].values()}
+            # load modules associate to this host
+            modules_name = set()
+            for name, mod_datas in datas['servers'].items():
+                if not 'module' in mod_datas:
+                    raise Exception(f'module is mandatory for "{name}"')
+                modules_name.add(mod_datas['module'])
+            # load modules informations from config files
             modules = Modules(datas['applicationservices'],
                               applicationservices,
                               datas['applicationservice_provider'],
                               modules_name,
                               self.servers_json['modules']
                               )
+
+            # load host
             module_info = modules.get('host')
             cfg['risotto_globals'][host_name] = {'global:server_name': host_name,
                                                  'global:module_name': 'host',
                                                  'global:host_install_dir': abspath(INSTALL_DIR),
                                                  }
             functions_files |= set(module_info.functions_file)
-            self.load_dictionaries(cfg, module_info, host_name, rougail)
+            self.load_dictionaries(cfg,
+                                   module_info,
+                                   host_name,
+                                   rougail,
+                                   )
+            # load servers
             modules_info = {}
             for server_name, server_datas in datas['servers'].items():
                 module_info = modules.get(server_datas['module'])
@@ -188,11 +257,15 @@ class Loader:
                                                      }
                 server_datas['server_name'] = values[0]
                 functions_files |= set(module_info.functions_file)
-                self.load_dictionaries(cfg, module_info, values[0], rougail)
+                self.load_dictionaries(cfg,
+                                       module_info,
+                                       values[0],
+                                       rougail,
+                                       )
                 modules_info[module_info.module_name] = module_info.depends
             self.modules[host_name] = modules_info
         cfg['functions_file'] = list(functions_files)
-        self.tiram_obj = rougail.save(self.cache_file)
+        self.tiram_obj = rougail.save(TIRAMISU_CACHE)
 
     def load_dictionaries(self, cfg, module_info, server_name, rougail):
         cfg['dictionaries_dir'] = module_info.dictionaries_dir
@@ -202,11 +275,14 @@ class Loader:
         self.templates_dir[server_name] = module_info.templates_dir
         self.patches_dir[server_name] = module_info.patches_dir
         self.functions_files[server_name] = module_info.functions_file
+        self.manuals_dirs[server_name] = module_info.manuals
+        self.tests_dirs[server_name] = module_info.tests
 
-    async def load(self):
+    async def tiramisu_file_to_tiramisu(self):
-        optiondescription = FUNCTIONS.copy()
+        # l
+        tiramisu_space = FUNCTIONS.copy()
         try:
-            exec(self.tiram_obj, None, optiondescription)
+            exec(self.tiram_obj, None, tiramisu_space)
         except Exception as err:
             print(self.tiram_obj)
             raise Exception(f'unknown error when load tiramisu object {err}') from err
@@ -214,12 +290,13 @@ class Loader:
             display_name = None
         else:
             display_name = tiramisu_display_name
-        self.config = await Config(optiondescription['option_0'],
+        self.config = await Config(tiramisu_space['option_0'],
                                    display_name=display_name,
                                    )
 
-    async def after(self):
+    async def load_values_and_informations(self):
         config = self.config
+        await config.property.read_write()
         await config.property.pop('validator')
         await config.property.pop('cache')
         load_zones(self.servers_json)
@@ -238,18 +315,27 @@ class Loader:
                 await information.set('templates_dir', self.templates_dir[server_name])
                 await information.set('patches_dir', self.patches_dir[server_name])
                 await information.set('functions_files', self.functions_files[server_name])
+                await information.set('manuals_dirs', self.manuals_dirs[server_name])
+                await information.set('tests_dirs', self.tests_dirs[server_name])
                 await self.set_values(server_name, config, datas)
 
+        await config.information.set('copy_tests', False)
         # FIXME only one host_name is supported
         await config.information.set('modules', self.modules[host_name])
         # await config.information.set('modules', {module_name: module_info.depends for module_name, module_info in self.module_infos.items() if module_name in modules})
-        await config.property.read_only()
         await config.property.add('cache')
         if self.valid_mandatories:
-            await valid_mandatories(config)
+            messages = await valid_mandatories(config)
-        with open(self.cache_values, 'w') as fh:
+            if messages:
+                msg = ''
+                for title, variables in messages.items():
+                    msg += '\n' + title + '\n'
+                    msg += '\n'.join(variables)
+                raise Exception(msg)
+        await config.property.read_only()
+        with open(VALUES_CACHE, 'w') as fh:
             json_dump(await config.value.exportation(), fh)
-        with open(self.cache_informations, 'w') as fh:
+        with open(INFORMATIONS_CACHE, 'w') as fh:
             json_dump(await config.information.exportation(), fh)
 
     async def set_values(self,
@@ -259,6 +345,8 @@ class Loader:
                          ):
         if 'values' not in datas:
             return
+        if not isinstance(datas['values'], dict):
+            raise Exception(f'Values of "{server_name}" are not a dict: {datas["values"]}')
         server_path = normalize_family(server_name)
         await config.owner.set(self.config_file)
         for vpath, value in datas['values'].items():
@@ -275,19 +363,16 @@ class Loader:
                 raise Exception(error_msg) from err
         await config.owner.set('user')
 
-    async def finish(self):
-        await self.config.property.read_only()
-
 
 class LoaderCache(Loader):
-    def before(self):
+    def load_tiramisu_file(self):
-        with open(self.cache_file) as fh:
+        with open(TIRAMISU_CACHE) as fh:
             self.tiram_obj = fh.read()
 
-    async def after(self):
+    async def load_values_and_informations(self):
-        with open(self.cache_values, 'r') as fh:
+        with open(VALUES_CACHE, 'r') as fh:
             await self.config.value.importation(json_load(fh))
-        with open(self.cache_informations, 'r') as fh:
+        with open(INFORMATIONS_CACHE, 'r') as fh:
             informations = json_load(fh)
             # null is not a valid key in json => 'null'
             informations[None] = informations.pop('null')
@@ -298,21 +383,22 @@ async def load(clean_directories=False,
                hide_secret=False,
                original_display_name: bool=False,
                valid_mandatories: bool=True,
+               copy_tests: bool=False,
                ):
     if isfile(TIRAMISU_CACHE) and isfile(VALUES_CACHE) and isfile(INFORMATIONS_CACHE):
         loader_obj = LoaderCache
     else:
         loader_obj = Loader
-    loader = loader_obj(TIRAMISU_CACHE,
+    loader = loader_obj(clean_directories,
-                        VALUES_CACHE,
-                        INFORMATIONS_CACHE,
-                        clean_directories,
                         hide_secret,
                         original_display_name,
                         valid_mandatories,
                         )
-    loader.before()
+    loader.load_tiramisu_file()
-    await loader.load()
+    await loader.tiramisu_file_to_tiramisu()
-    await loader.after()
+    await loader.load_values_and_informations()
-    await loader.finish()
+    config = loader.config
-    return loader.config
+    await config.property.read_only()
+    await config.information.set('copy_tests', copy_tests)
+    await config.cache.reset()
+    return config

src/risotto/rougail/annotator.py

@@ -135,8 +135,6 @@ class Annotator(Walk):
                  objectspace: 'RougailObjSpace',
                  *args):
         self.objectspace = objectspace
-#        self.convert_get_linked_information()
-#        self.convert_provider()
         self.set_suppliers()
         self.convert_providers()
         self.convert_suppliers()
@@ -159,28 +157,9 @@ class Annotator(Walk):
                                                                      'zone_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'],
                                                                      'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'])
                                                                      })
-
+            if not hasattr(variable, 'information'):
-    def convert_suppliers(self):
+                variable.information = self.objectspace.information(variable.xmlfiles)
-        for supplier, data in self.suppliers.items():
+            variable.information.supplier = variable.supplier
-            if supplier == 'Host':
-                continue
-            for s_dico in data:
-                if supplier not in self.providers:
-                    continue
-                for p_dico in self.providers[supplier]:
-                    common_zones = s_dico['zones'] & p_dico['zones']
-                    if common_zones:
-                        for idx, zone in enumerate(p_dico['zone_names']):
-                            if zone in common_zones:
-                                break
-                        dns = p_dico['server_names'][idx]
-#                        dns = p_dico["dns"]
-                        s_dico['option'].value = dns
-                        new_value = self.objectspace.value(None)
-                        new_value.name = dns
-                        s_dico['option'].value = [new_value]
-                        break
-
 
     def convert_providers(self):
         self.providers = {}
@@ -194,6 +173,16 @@ class Annotator(Walk):
                 server_names = [server_name]
             else:
                 server_names = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names']
+            if provider_name != 'Host' and not provider_name.startswith('Host:') and not provider_name.startswith('global:'):
+                p_data = {'option': variable,
+                          'dns': server_name,
+                          'path_prefix': nf_dns,
+                          'server_names': server_names,
+                          'zone_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'],
+                          'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name']),
+                          }
+            else:
+                p_data = None
             if ':' in provider_name:
                 key_name, key_type = provider_name.rsplit(':', 1)
                 is_provider = False
@@ -201,13 +190,7 @@ class Annotator(Walk):
                 key_name = key_type = provider_name
                 is_provider = True
                 if provider_name != 'Host':
-                    self.providers.setdefault(provider_name, []).append({'option': variable,
+                    self.providers.setdefault(provider_name, []).append(p_data)
-                                                                         'dns': server_name,
-                                                                         'path_prefix': nf_dns,
-                                                                         'server_names': server_names,
-                                                                         'zone_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'],
-                                                                         'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name']),
-                                                                         })
             if key_name != 'global' and key_name not in self.suppliers:
                 #warn(f'cannot find supplier "{key_name}" for "{server_name}"')
                 continue
@@ -270,16 +253,30 @@ class Annotator(Walk):
                 fill.param.append(param)
             if key_name == 'global':
                 param = self.objectspace.param(variable.xmlfiles)
-                if provider_name not in self.objectspace.rougailconfig['risotto_globals'][server_name]:
-                    raise DictConsistencyError(f'cannot find provider "{provider_name}" for variable "{variable.name}"', 200, variable.xmlfiles)
-                param.text = self.objectspace.rougailconfig['risotto_globals'][server_name][provider_name]
                 param.name = 'value'
+                if provider_name in self.objectspace.rougailconfig['risotto_globals'][server_name]:
+                    value = self.objectspace.rougailconfig['risotto_globals'][server_name][provider_name]
+                    param.text = value
+                    if isinstance(value, bool):
+                        param.type = 'boolean'
+                else:
+                    param.text = provider_name
+                    param.type = 'information'
                 fill.param.append(param)
             else:
                 # parse all supplier link to current provider
                 for idx, data in enumerate(self.suppliers[key_name]):
-                    option = data['option']
+                    if p_data:
+                        common_zones = data['zones'] & p_data['zones']
+                        if not common_zones:
+                            continue
+                        for zidx, zone in enumerate(data['zone_names']):
+                            if zone in common_zones:
+                                break
+                        dns = data['server_names'][zidx]
+                    else:
                         dns = data['dns']
+                    option = data['option']
                     # if not provider, get the true option that we want has value
                     if not is_provider:
                         path_prefix = data['path_prefix']
@@ -330,3 +327,24 @@ class Annotator(Walk):
             if not hasattr(self.objectspace.space.variables[nf_dns].constraints, 'fill'):
                 self.objectspace.space.variables[nf_dns].constraints.fill = []
             self.objectspace.space.variables[nf_dns].constraints.fill.append(fill)
+
+    def convert_suppliers(self):
+        for supplier, data in self.suppliers.items():
+            if supplier == 'Host':
+                continue
+            for s_dico in data:
+                if supplier not in self.providers:
+                    continue
+                for p_dico in self.providers[supplier]:
+                    common_zones = s_dico['zones'] & p_dico['zones']
+                    if not common_zones:
+                        continue
+                    for idx, zone in enumerate(p_dico['zone_names']):
+                        if zone in common_zones:
+                            break
+                    dns = p_dico['server_names'][idx]
+                    s_dico['option'].value = dns
+                    new_value = self.objectspace.value(None)
+                    new_value.name = dns
+                    s_dico['option'].value = [new_value]
+                    break