reorganise

ansible/action_plugins/build_images.py

@@ -13,36 +13,46 @@ class ActionModule(ActionBase):
         super(ActionModule, self).run(tmp, task_vars)
         module_args = self._task.args.copy()
         modules = module_args['modules']
-        dataset_directory = RISOTTO_CONFIG['directories']['dataset']
+        dataset_directories = RISOTTO_CONFIG['directories']['datasets']
         install_dir = join('/tmp/risotto/images')
         if isdir(install_dir):
             rmtree(install_dir)
         for module_name, depends in modules.items():
-            for depend in depends:
-                manual = join(dataset_directory, depend, 'manual', 'image')
-                if not isdir(manual):
-                    continue
-                for filename in listdir(manual):
-                    src_file = join(manual, filename)
-                    dst_file = join(install_dir, module_name, filename)
-                    if isdir(src_file):
-                        if not isdir(dst_file):
-                            makedirs(dst_file)
-                        for subfilename in listdir(src_file):
-                            if not isfile(dst_file):
-                                src = join(src_file, subfilename)
-                                dst = join(dst_file, subfilename)
-                                if isfile(src):
-                                    copy2(src, dst)
-                                else:
-                                    copytree(src, dst)
-                    elif not isfile(dst_file):
-                        dst = dirname(dst_file)
-                        if not isdir(dst):
-                            makedirs(dst)
-                        if isfile(src_file):
-                            copy2(src_file, dst_file)
-                        else:
-                            copytree(src_file, dst_file)
+            for dataset_directory in dataset_directories:
+                for depend in depends:
+                    manual = join(dataset_directory, depend, 'manual', 'image')
+                    if not isdir(manual):
+                        continue
+                    for filename in listdir(manual):
+                        src_file = join(manual, filename)
+                        dst_file = join(install_dir, module_name, filename)
+                        if isdir(src_file):
+                            if not isdir(dst_file):
+                                makedirs(dst_file)
+                            for subfilename in listdir(src_file):
+                                if not isfile(dst_file):
+                                    src = join(src_file, subfilename)
+                                    dst = join(dst_file, subfilename)
+                                    if isfile(src):
+                                        copy2(src, dst)
+                                    else:
+                                        copytree(src, dst)
+                        elif not isfile(dst_file):
+                            dst = dirname(dst_file)
+                            if not isdir(dst):
+                                makedirs(dst)
+                            if isfile(src_file):
+                                copy2(src_file, dst_file)
+                            else:
+                                copytree(src_file, dst_file)
         return dict(ansible_facts=dict({}))
-
+#A REFAIRE ICI    tests_dir = join(as_dir, 'tests')
+#A REFAIRE ICI    if isdir(tests_dir):
+#A REFAIRE ICI        cfg.tests.append(tests_dir)
+#A REFAIRE ICI#        for filename in listdir(tests_dir):
+#A REFAIRE ICI#            src_file = join(tests_dir, filename)
+#A REFAIRE ICI#            dst_file = join(INSTALL_DIR, 'tests', filename)
+#A REFAIRE ICI#            applicationservice_copy(src_file,
+#A REFAIRE ICI#                                    dst_file,
+#A REFAIRE ICI#                                    False,
+#A REFAIRE ICI#                                    )

ansible/action_plugins/rougail.py

@@ -1,46 +1,29 @@
 #!/usr/bin/python3
-from ansible.plugins.action import ActionBase
 from asyncio import run
-from shutil import rmtree
-from os.path import isdir, join
-from os import makedirs
-
-from risotto.machine import templates, load, ROUGAIL_NAMESPACE
-from risotto.utils import RISOTTO_CONFIG
-from rougail.utils import normalize_family
+from os import readlink
+from os.path import join, islink
+from risotto.machine import load, templates
+try:
+    from ansible.plugins.action import ActionBase
+    from ansible.module_utils.basic import AnsibleModule
+    class FakeModule(AnsibleModule):
+        def __init__(self):
+            pass
+except:
+    import traceback
+    traceback.print_exc()
+    class ActionBase():
+        def __init__(self, *args, **kwargs):
+            raise Exception('works only with ansible')
 
 
-TIRAMISU_CACHE = 'tiramisu_cache.py'
-VALUES_CACHE = 'values_cache.py'
-INSTALL_DIR = RISOTTO_CONFIG['directories']['dest']
-
-
-async def build_files(server_name, is_host):
-    module_infos, rougailconfig, config = await load(TIRAMISU_CACHE,
-                                                     VALUES_CACHE,
-                                                     )
-    subconfig = config.option(normalize_family(server_name))
-    module_name = await subconfig.option(await subconfig.information.get('provider:global:module_name')).value.get()
-    module_info = module_infos[module_name]
-    rougailconfig['tmp_dir'] = 'tmp'
-    rougailconfig['destinations_dir'] = INSTALL_DIR
-    rougailconfig['templates_dir'] = module_info['infos'].templates_dir
-    if is_host:
-        tmpfile = await subconfig.option(f'{ROUGAIL_NAMESPACE}.host_install_dir').value.get()
-        rougailconfig['tmpfile_dest_dir'] = f'{tmpfile}'
-        rougailconfig['default_systemd_directory'] = '/usr/local/lib/systemd'
-    else:
-        rougailconfig['tmpfile_dest_dir'] = '/usr/local/lib'
-        rougailconfig['default_systemd_directory'] = '/systemd'
-    if isdir(rougailconfig['destinations_dir']):
-        rmtree(rougailconfig['destinations_dir'])
-    if isdir(rougailconfig['tmp_dir']):
-        rmtree(rougailconfig['tmp_dir'])
-    makedirs(rougailconfig['tmp_dir'])
-    makedirs(rougailconfig['destinations_dir'])
+async def build_files(server_name: str,
+                      just_copy: bool,
+                      ) -> None:
+    config = await load()
     await templates(server_name,
-                    subconfig,
-                    rougailconfig,
+                    config,
+                    just_copy=just_copy,
                     )
 
 
@@ -48,8 +31,38 @@ class ActionModule(ActionBase):
     def run(self, tmp=None, task_vars=None):
         super(ActionModule, self).run(tmp, task_vars)
         module_args = self._task.args.copy()
-        name = module_args['hostname']
-        is_host = module_args['is_host']
+        root_local = module_args.pop('root_local')
+        root_remote= module_args.pop('root_remote')
+        name = module_args.pop('hostname')
+        is_host = module_args.pop('is_host')
+        just_copy = module_args.get('just_copy', False)
+        module_args['root'] = root_remote
 
-        run(build_files(name, is_host))
-        return dict(ansible_facts=dict({}))
+        run(build_files(name, just_copy))
+        #
+        remote = self._execute_module(module_name='compare', module_args=module_args, task_vars=task_vars)
+        if remote.get('failed'):
+            raise Exception(f'error in remote action: {remote["module_stdout"]}')
+        #
+        module = FakeModule()
+        modified_files = []
+        changed = False
+        for path in module_args['paths']:
+            full_path = join(root_local, path['name'][1:])
+            if remote['compare'].get(path['name']):
+                if remote['compare'][path['name']]['type'] == 'file':
+                    if remote['compare'][path['name']]['shasum'] == module.digest_from_file(full_path, 'sha256'):
+                        continue
+                else:
+                    # it's a symlink
+                    if islink(full_path) and remote['compare'][path['name']]['name'] == readlink(full_path):
+                        continue
+            changed = True
+            modified_files.append(path['name'])
+        if not is_host:
+            for old_file in remote['old_files']:
+                changed = True
+    #            module_args['path'] = old_file
+    #            module_args['state'] = 'absent'
+    #            self._execute_module(module_name='ansible.builtin.file', module_args=module_args, task_vars=task_vars)
+        return dict(ansible_facts=dict({}), changed=changed)

ansible/filter_plugins/fileslist.py

@@ -31,22 +31,30 @@ def fileslist(data, is_host=False, name_only=False, prefix=None):
              )
     for service, service_data in data.items():
         if not service_data['activate']:
-            if service_data['engine'] == 'none' and service_data['type'] == 'service' and not 'overrides' in service_data:
-                _add(files,
-                     {'owner': 'root', 'group': 'root', 'mode': '0755'},
-                     base_systemd + '/systemd/system/' + service_data['doc'],
-                     name_only,
-                     prefix,
-                     )
+            if service_data['manage']:
+                if not service_data.get('undisable', False) and not service_data['engine'] and not service_data.get('target'):
+                    _add(files,
+                         {'owner': 'root', 'group': 'root', 'mode': '0755'},
+                         base_systemd + '/systemd/system/' + service_data['doc'],
+                         name_only,
+                         prefix,
+                         )
         else:
-            if service_data['activate'] and service_data['engine'] != 'none':
+            if service_data['manage'] and service_data['engine']:
                 _add(files,
                      {'owner': 'root', 'group': 'root', 'mode': '0755'},
                      base_systemd + '/systemd/system/' + service_data['doc'],
                      name_only,
                      prefix,
                      )
-            if service_data['activate'] and 'overrides' in service_data:
+            if service_data.get('target'):
+                _add(files,
+                     {'owner': 'root', 'group': 'root', 'mode': '0755'},
+                     f'/systemd/system/{service_data["target"]}.target.wants/{service_data["doc"]}',
+                     name_only,
+                     prefix,
+                     )
+            if 'overrides' in service_data:
                 for override_data in service_data['overrides'].values():
                     _add(files,
                          {'owner': 'root', 'group': 'root', 'mode': '0755'},
@@ -54,6 +62,13 @@ def fileslist(data, is_host=False, name_only=False, prefix=None):
                          name_only,
                          prefix,
                          )
+            if 'ip' in service_data:
+                _add(files,
+                     {'owner': 'root', 'group': 'root', 'mode': '0755'},
+                     base_systemd + '/systemd/system/' + service_data['doc'] + '.d/rougail_ip.conf',
+                     name_only,
+                     prefix,
+                     )
         if 'files' not in service_data:
             continue
         for file_data in service_data['files'].values():
@@ -88,6 +103,8 @@ def directorieslist(data):
 def machineslist(data, only=None, only_name=False):
     srv = []
     if only is not None:
+        if only not in data:
+            raise Exception(f"cannot find {only} but only {data.keys()}")
         if only_name:
             srv.append(only)
         else:

ansible/host.yml

@@ -2,6 +2,36 @@
 - name: "Populate service facts"
   service_facts:
 
+- name: "Packages installation"
+  apt:
+    pkg: "{{ vars[inventory_hostname]['general']['host_packages'] }}"
+    update_cache: yes
+    state: latest
+
+- name: "Build host files"
+  rougail:
+    paths: "{{ vars[inventory_hostname]['services'] | fileslist(is_host=True) }}"
+    root_local: "{{ host_install_dir }}"
+    root_remote: "/"
+    hostname: "{{ inventory_hostname }}"
+    is_host: True
+
+- name: "Create /usr/local/lib/systemd/system"
+  file:
+    path: /usr/local/lib/systemd/system
+    state: directory
+    mode: 0755
+
+- name: "Copy service file only if not exists"
+  when: item.value['manage'] and item.value['activate'] and item.value['doc'].endswith('.service') and not item.value['doc'].endswith('@.service') and item.value['engine'] and item.value['engine'] != 'none'
+  copy:
+    src: '{{ host_install_dir }}/usr/local/lib/systemd/system/{{ item.value["doc"] }}'
+    force: no
+    dest: '/usr/local/lib/systemd/system/{{ item.value["doc"] }}'
+  loop: "{{ vars[inventory_hostname]['services'] | dict2items }}"
+  loop_control:
+    label: "{{ item.value['doc'] }}"
+
 - name: "Stop services"
   when: item.value['manage'] and item.value['activate'] and item.value['doc'].endswith('.service') and not item.value['doc'].endswith('@.service') and item.value['engine'] != 'none'
   ansible.builtin.service:
@@ -11,18 +41,6 @@
   loop_control:
     label: "{{ item.value['doc'] }}"
 
-- name: "Packages installation"
-  apt:
-    pkg: "{{ vars[inventory_hostname]['general']['host_packages'] }}"
-    update_cache: yes
-    state: latest
-
-- name: "Build host files"
-  local_action:
-    module: rougail
-    hostname: "{{ inventory_hostname }}"
-    is_host: True
-
 - name: "Create host directories"
   file: path={{ item }} state=directory mode=0755
   loop: "{{ vars[inventory_hostname]['services'] | directorieslist }}"
@@ -30,7 +48,7 @@
 - name: "Copy systemd-tmpfiles"
   when: item.name.startswith('/usr/local/lib/risotto-tmpfiles.d')
   ansible.builtin.copy:
-    src: installations/{{ item.name }}
+    src: "{{ host_install_dir }}/{{ item.name }}"
     dest: "{{ item.name }}"
     owner: "{{ item.owner }}"
     group: "{{ item.group }}"
@@ -49,7 +67,7 @@
 - name: "Copy host files"
   when: not item.name.startswith('/usr/local/lib/tmpfiles.d')
   ansible.builtin.copy:
-    src: installations/{{ item.name }}
+    src: "{{ host_install_dir }}/{{ item.name }}"
     dest: "{{ item.name }}"
     owner: "{{ item.owner }}"
     group: "{{ item.group }}"
@@ -105,7 +123,7 @@
     owner: "root"
     group: "root"
     mode: "0755"
-  loop: "{{ lookup('fileglob', '../sbin/*', wantlist=True) | list }}"
+  loop: "{{ lookup('fileglob', 'sbin/*', wantlist=True) | list }}"
 
 # Images informations
 - name: "Remove images tar"
@@ -141,3 +159,9 @@
   unarchive:
     src: "/tmp/risotto/images.tar"
     dest: "/var/lib/risotto/images_files"
+
+- name: "Create versions directory"
+  file:
+    path: /var/lib/risotto/machines_versions
+    state: directory
+    mode: "0700"

ansible/inventory.py

@@ -5,21 +5,21 @@ Example custom dynamic inventory script for Ansible, in Python.
 '''
 
 from argparse import ArgumentParser
-from json import dumps, JSONEncoder
+from json import load as json_load, dumps, JSONEncoder
 from os import remove
 from os.path import isfile
 from asyncio import run
+from traceback import print_exc
 
-from risotto.machine import load
-from risotto.image import load_config
-from risotto.utils import SERVERS
+from risotto.machine import load, TIRAMISU_CACHE, VALUES_CACHE, INFORMATIONS_CACHE, ROUGAIL_NAMESPACE, ROUGAIL_NAMESPACE_DESCRIPTION
+from tiramisu import Config
 from tiramisu.error import PropertiesOptionError
 from rougail.utils import normalize_family
-from rougail import RougailSystemdTemplate
+from rougail import RougailSystemdTemplate, RougailConfig
 from rougail.template.base import RougailLeader, RougailExtra
 
-TIRAMISU_CACHE = 'tiramisu_cache.py'
-VALUES_CACHE = 'values_cache.py'
+
+DEBUG = False
 
 
 class RougailEncoder(JSONEncoder):
@@ -38,36 +38,43 @@ class RisottoInventory(object):
         parser = ArgumentParser()
         parser.add_argument('--list', action='store_true')
         parser.add_argument('--host', action='store')
+        parser.add_argument('--nocache', action='store_true')
+        parser.add_argument('--debug', action='store_true')
         self.args = parser.parse_args()
+        if self.args.debug:
+            global DEBUG
+            DEBUG = True
 
     async def run(self):
-        if self.args.list:
+        if self.args.list and self.args.host:
+            raise Exception('cannot have --list and --host together')
+        if self.args.list or self.args.nocache:
             if isfile(TIRAMISU_CACHE):
                 remove(TIRAMISU_CACHE)
             if isfile(VALUES_CACHE):
                 remove(VALUES_CACHE)
-            return await self.do_inventory()
+            if isfile(INFORMATIONS_CACHE):
+                remove(INFORMATIONS_CACHE)
+        config = await load(TIRAMISU_CACHE,
+                            VALUES_CACHE,
+                            INFORMATIONS_CACHE,
+                            )
+        if self.args.list:
+            return await self.do_inventory(config)
         elif self.args.host:
-            return await self.get_vars(self.args.host)
+            return await self.get_vars(config, self.args.host)
         raise Exception('pfff')
 
-    async def do_inventory(self):
-        module_infos = load_config(True,
-                                   True,
-                                   True,
-                                   )
-        servers = []
-        for server_name, server in SERVERS.items():
-            module_name = server['module']
-            if module_name != 'host':
-                continue
-            servers.append(server_name)
+    async def do_inventory(self,
+                           config: Config,
+                           ) -> dict:
+        servers = [await subconfig.option.doc() for subconfig in await config.option.list('optiondescription') if await subconfig.information.get('module') == 'host']
         return dumps({
             'group': {
                 'hosts': servers,
                 'vars': {
                     # FIXME
-                    'ansible_ssh_host': '192.168.56.156',
+#                    'ansible_ssh_host': '192.168.0.100',
                     'ansible_ssh_user': 'root',
                     'ansible_python_interpreter': '/usr/bin/python3'
                 }
@@ -75,41 +82,42 @@ class RisottoInventory(object):
         })
 
     async def get_vars(self,
+                       config: Config,
                        host_name: str,
                        ) -> dict:
-        try:
-            module_infos, rougailconfig, config = await load(TIRAMISU_CACHE,
-                                                             VALUES_CACHE,
-                                                             )
-        except Exception as err:
-    #        import traceback
-    #        traceback.print_exc()
-            print(err)
-            exit(1)
         ret = {}
-        modules = set()
-        for server_name, server in SERVERS.items():
-            if server['module'] == 'host' and server_name != host_name:
+        rougailconfig = RougailConfig.copy()
+        rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE
+        rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
+        for subconfig in await config.option.list('optiondescription'):
+            server_name = await subconfig.option.description() 
+            module_name = await subconfig.option(await subconfig.information.get('provider:global:module_name')).value.get()
+            if module_name == 'host' and server_name != host_name:
                 continue
-            modules.add(server['module'])
-            subconfig = config.option(normalize_family(server_name))
             engine = RougailSystemdTemplate(subconfig, rougailconfig)
             await engine.load_variables()
-            if server['module'] != 'host' and engine.rougail_variables_dict['general']['host'] != host_name:
+            if module_name != 'host' and engine.rougail_variables_dict['general']['host'] != host_name:
                 continue
             ret[server_name] = engine.rougail_variables_dict
-        ret['modules'] = {module_name: module_info['infos'].depends for module_name, module_info in module_infos.items() if module_name in modules}
+        ret['modules'] = await config.information.get('modules')
         ret['delete_old_image'] = False
         ret['configure_host'] = True
         ret['only_machine'] = None
+        ret['copy_template'] = False
+        ret['host_install_dir'] = ret[host_name].pop('host_install_dir')
         return dumps(ret, cls=RougailEncoder)
 
       
 # Get the inventory.
 async def main():
-    inv = RisottoInventory()
-    values = await inv.run()
-    print(values)
+    try:
+        inv = RisottoInventory()
+        values = await inv.run()
+        print(values)
+    except Exception as err:
+        if DEBUG:
+            print_exc()
+        exit(err)
 
 
 run(main())

ansible/library/compare.py

@@ -0,0 +1,68 @@
+#!/usr/bin/python3
+
+from time import sleep
+from os import fdopen, walk, readlink
+from os.path import join, islink
+from dbus import SystemBus, Array
+from dbus.exceptions import DBusException
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def run_module():
+    # define available arguments/parameters a user can pass to the module
+    module_args = dict(
+        root=dict(type='str', required=True),
+        paths=dict(type='list', required=True),
+    )
+
+    # seed the result dict in the object
+    # we primarily care about changed and state
+    # changed is if this module effectively modified the target
+    # state will include any data that you want your module to pass back
+    # for consumption, for example, in a subsequent task
+    result = dict(
+        changed=False,
+        compare={},
+        symlink={},
+        old_files=[],
+    )
+
+    # the AnsibleModule object will be our abstraction working with Ansible
+    # this includes instantiation, a couple of common attr would be the
+    # args/params passed to the execution, as well as if the module
+    # supports check mode
+    module = AnsibleModule(
+        argument_spec=module_args,
+        supports_check_mode=True
+    )
+
+    root = module.params['root']
+    if root != '/':
+        paths = {join(root, path['name'][1:]): path['name'] for path in module.params['paths']}
+        search_paths = [join(directory, f) for directory, subdirectories, files in walk(root) for f in files]
+    else:
+        paths = {path['name']: path['name'] for path in module.params['paths']}
+        search_paths = paths
+    for path in search_paths:
+        if path in paths:
+            if not islink(path):
+                result['compare'][paths[path]] = {'type': 'file',
+                                                  'shasum': module.digest_from_file(path, 'sha256'),
+                                                  }
+            else:
+                result['compare'][paths[path]] = {'type': 'symlink',
+                                                  'name': readlink(path),
+                                                  }
+        else:
+            result['old_files'].append(path)
+
+    module.exit_json(**result)
+
+
+def main():
+    run_module()
+
+
+if __name__ == '__main__':
+    main()

ansible/library/machinectl.py

@@ -82,6 +82,7 @@ def start(bus, machines):
             error = False
             while True:
                 try:
+                    ret = []
                     res = remote_object.OpenMachineShell(host,
                                                          '',
                                                          cmd[0],
@@ -91,7 +92,6 @@ def start(bus, machines):
                                                          )
                     fd = res[0].take()
                     fh = fdopen(fd)
-                    ret = []
                     while True:
                         try:
                             ret.append(fh.readline().strip())
@@ -114,7 +114,7 @@ def start(bus, machines):
                     break
             if error:
                 continue
-            if ret[0] == 'running':
+            if ret and ret[0] == 'running':
                 continue
             cmd = ['/usr/bin/systemctl', '--state=failed', '--no-legend', '--no-page']
             res = remote_object.OpenMachineShell(host,

ansible/machine.yml

@@ -6,65 +6,69 @@
   file: path=/var/lib/risotto/journals/{{ item.name }} state=directory mode=0755
 
 - name: "Build machine files for {{ item.name }}"
-  local_action:
-    module: rougail
+  rougail:
+    paths: "{{ vars[item.name]['services'] | fileslist }}"
+    root_local: "{{ host_install_dir }}"
+    root_remote: "/var/lib/risotto/configurations/{{ item.name }}"
     hostname: "{{ item.name}}"
     is_host: False
-
-- name: "Get local informations for {{ item.name }} configuration's file"
-  local_action:
-    module: stat
-    path: "installations{{ file.name }}"
-    checksum: sha256
-    get_checksum: yes
-    follow: true
-  loop: "{{ vars[item.name]['services'] | fileslist }}"
-  loop_control:
-    loop_var: file
-    label: "{{ file.name }}"
-  register: local_configuration
-
-- name: "Get remote informations for {{ item.name }} configuration's file"
-  stat:
-    path: "/var/lib/risotto/configurations/{{ item.name }}{{ file.name }}"
-    checksum: sha256
-    get_checksum: yes
-    follow: true
-  loop: "{{ vars[item.name]['services'] | fileslist }}"
-  loop_control:
-    loop_var: file
-    label: "{{ file.name }}"
-  register: remote_configuration
-
-- name: "Configuration's file is up to date in {{ item.name }}"
-  debug:
-    msg: "file is {{ 'out of date' if not file[1].stat.exists or (not 'checksum' in file[0].stat and 'checksum' in file[1].stat) or ('checksum' in file[0].stat and not 'checksum' in file[1].stat) or ('checksum' in file[0].stat and 'checksum' in file[1].stat and file[0].stat.checksum != file[1].stat.checksum) else 'up to date' }}"
-  changed_when: not file[1].stat.exists or (not 'checksum' in file[0].stat and 'checksum' in file[1].stat) or ('checksum' in file[0].stat and not 'checksum' in file[1].stat) or ('checksum' in file[0].stat and 'checksum' in file[1].stat and file[0].stat.checksum != file[1].stat.checksum)
-  loop: "{{ local_configuration.results | zip(remote_configuration.results) | list }}"
-  loop_control:
-    loop_var: file
-    label: "{{ file[0]['file']['name'] }}"
-  ignore_errors: true
   register: up_to_date_configuration
 
-- name: "Remove Compressed files for {{ item.name }}"
+- name: "Change secrets right"
   local_action:
     module: file
-    path: /tmp/new_configurations/{{ item.name }}
-    state: absent
-  when: up_to_date_configuration.changed
+    path: "{{ host_install_dir }}/secrets"
+    state: directory
+    mode: 0700
 
 - name: "Compress files for {{ item.name }}"
   local_action:
     module: archive
-    path: "installations/"
+    path: "{{ host_install_dir }}/"
     dest: /tmp/new_configurations/{{ item.name }}
     format: tar
   when: up_to_date_configuration.changed
 
+- name: "Build machine templates for {{ item.name }}"
+  rougail:
+    paths: "{{ vars[item.name]['services'] | fileslist }}"
+    root_local: "{{ host_install_dir }}"
+    root_remote: "/var/lib/risotto/configurations/{{ item.name }}"
+    hostname: "{{ item.name}}"
+    just_copy: true
+    is_host: False
+  when: copy_template
+  register: up_to_date_configuration
+
+- name: "Compress templates for {{ item.name }}"
+  local_action:
+    module: archive
+    path: "../templates/"
+    dest: /tmp/new_templates/{{ item.name }}
+    format: tar
+  when: copy_template
+
+- name: "Remove templates directory for {{ item.name }}"
+  file:
+    path: "/var/lib/risotto/templates/{{ item.name }}"
+    state: absent
+  when: copy_template
+
+- name: "Create templates directory for {{ item.name }}"
+  file:
+    path: "/var/lib/risotto/templates/{{ item.name }}"
+    state: directory
+  when: copy_template
+
+- name: "Copy templates for {{ item.name }}"
+  unarchive:
+    src: "/tmp/new_templates/{{ item.name }}"
+    dest: "/var/lib/risotto/templates/{{ item.name }}/"
+  when: copy_template
+
 - name: "Remove old image {{ vars | modulename(item.name) }}"
   file:
-    path: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}.tar"
+    path: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}"
     state: absent
   when: delete_old_image == true
 
@@ -88,27 +92,30 @@
 
 - name: "Check image for {{ item.name }}"
   stat:
-    path: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}.tar"
+    path: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}"
     follow: true
   register: register_name
   when: system_directory_created.changed
 
+#- name: Print return information from the previous task
+#  ansible.builtin.debug:
+#    var: register_name
+
 - name: "Build image for {{ item.name }}"
   ansible.builtin.shell: "/usr/local/sbin/build_image {{ vars | modulename(item.name) }}"
-  when: system_directory_created.changed and not register_name.stat.exists
+  when: "'stat' in register_name and not register_name.stat.exists"
+  register: ret
+  failed_when: ret.rc != 0
 
-- name: "Uncompress machine image for {{ item.name }}"
-  unarchive:
-    src: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}.tar"
-    remote_src: true
-    dest: /var/lib/machines/{{ item.name }}/
+- name: "Copy machine image for {{ item.name }}"
+  ansible.builtin.shell: "/usr/bin/cp -a --reflink=auto /var/lib/risotto/images/{{ vars | modulename(item.name) }}/* /var/lib/machines/{{ item.name }}"
   when: system_directory_created.changed
 
-- name: "SHA machine image for {{ item.name }}"
+- name: "Copy machine image version for {{ item.name }}"
   ansible.builtin.copy:
-    src: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}.tar.sha"
+    src: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}.version"
     remote_src: true
-    dest: "/var/lib/risotto/configurations/sha/{{ item.name }}.sha"
+    dest: "/var/lib/risotto/machines_versions/{{ item.name }}.version"
     owner: "root"
     group: "root"
   when: system_directory_created.changed

ansible/machines.yml

@@ -19,6 +19,8 @@
   unarchive:
     src: "{{ item }}"
     dest: /var/lib/risotto/configurations/{{ item | basename }}/
+    owner: root
+    group: root
   loop: "{{ lookup('fileglob', '/tmp/new_configurations/*', wantlist=True) }}"
 
 - name: "Enable machines"
@@ -36,3 +38,9 @@
     module: file
     path: /tmp/new_configurations
     state: absent
+
+- name: "Remove compressed templates directory"
+  local_action:
+    module: file
+    path: /tmp/new_templates
+    state: absent

ansible/playbook.yml

@@ -1,7 +1,6 @@
 ---
-#FIXME : si on redemarre a appel tmpfiles.d ....
 - name: Risotto
-  hosts: cloud.silique.fr
+  hosts: all
   tasks:
     - name: "Configure the host"
       include_tasks: host.yml
@@ -20,6 +19,21 @@
         state: directory
         mode: 0700
     
+    - name: "Remove compressed templates files directory"
+      local_action:
+        module: file
+        path: /tmp/new_templates
+        state: absent
+      when: copy_template
+    
+    - name: "Create compressed templates files directory"
+      local_action:
+        module: file
+        path: /tmp/new_templates
+        state: directory
+        mode: 0700
+      when: copy_template
+    
     - name: "Prepare machine configuration"
       include_tasks: machine.yml
       loop: "{{ vars | machineslist(only=only_machine) }}"

ansible/sbin/build_image

@@ -12,15 +12,13 @@ RISOTTO_DIR="/var/lib/risotto"
 RISOTTO_IMAGE_DIR="$RISOTTO_DIR/images"
 # image configuration
 IMAGE_BASE_RISOTTO_BASE_DIR="$RISOTTO_IMAGE_DIR/image_bases"
+IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP="$RISOTTO_IMAGE_DIR/tmp/$IMAGE_NAME"
 IMAGE_NAME_RISOTTO_IMAGE_DIR="$RISOTTO_IMAGE_DIR/$IMAGE_NAME"
-IMAGE_NAME_RISOTTO_IMAGE_NAME="$RISOTTO_IMAGE_DIR/$IMAGE_NAME".tar
 IMAGE_DIR_RECIPIENT_IMAGE="/var/lib/risotto/images_files/$IMAGE_NAME"
 
 
-#FIXME ou ?
-
-rm -rf "$IMAGE_NAME_RISOTTO_IMAGE_DIR" "$RISOTTO_IMAGE_DIR/tmp"
-mkdir -p "$RISOTTO_IMAGE_DIR"
+rm -f /var/log/risotto/build_image.log
+mkdir -p "$RISOTTO_IMAGE_DIR" "$RISOTTO_IMAGE_DIR/tmp/" /var/log/risotto
 PKG=""
 BASE_DIR=""
 for script in $(ls "$IMAGE_DIR_RECIPIENT_IMAGE"/preinstall/*.sh 2> /dev/null); do
@@ -29,93 +27,89 @@ done
 
 if [ -z "$OS_NAME" ]; then
     echo "NO OS NAME DEFINED"
-    exit 0
+    exit 1
 fi
 if [ -z "$RELEASEVER" ]; then
     echo "NO RELEASEVER DEFINED"
-    exit 0
+    exit 1
 fi
 if [ -z "$INSTALL_TOOL" ]; then
     echo "NO INSTALL TOOL DEFINED"
-    exit 0
+    exit 1
 fi
 BASE_NAME="$OS_NAME-$RELEASEVER"
 BASE_DIR="$IMAGE_BASE_RISOTTO_BASE_DIR/$BASE_NAME"
-BASE_TAR="$IMAGE_BASE_RISOTTO_BASE_DIR-$BASE_NAME".tar
+TMP_BASE_DIR="$IMAGE_BASE_RISOTTO_BASE_DIR/tmp/$BASE_NAME"
 BASE_PKGS_FILE="$IMAGE_BASE_RISOTTO_BASE_DIR-$BASE_NAME.pkgs"
 BASE_LOCK="$IMAGE_BASE_RISOTTO_BASE_DIR-$BASE_NAME.build"
 
 
 function dnf_opt_base() {
     INSTALL_DIR=$1
-    echo "--setopt=install_weak_deps=False --nodocs --noplugins --installroot=$INSTALL_DIR --releasever $RELEASEVER"
+    echo "--setopt=install_weak_deps=False --setopt=fastestmirror=True --nodocs --noplugins --installroot=$INSTALL_DIR --releasever $RELEASEVER"
 }
 
 function dnf_opt() {
     INSTALL_DIR=$1
     INSTALL_PKG=$2
-	OPT=$(dnf_opt_base "$INSTALL_DIR")
+    OPT=$(dnf_opt_base "$INSTALL_DIR")
     echo "$OPT install $INSTALL_PKG"
 }
 function new_package_base() {
     if [ "$INSTALL_TOOL" = "dnf" ]; then
-        OPT=$(dnf_opt "$BASE_DIR" "$BASE_PKG")
+        OPT=$(dnf_opt "$TMP_BASE_DIR" "$BASE_PKG")
         dnf --assumeno $OPT | grep ^" " > "$BASE_PKGS_FILE".new
     else
-        debootstrap --include="$BASE_PKG" --variant=minbase "$RELEASEVER" "$BASE_DIR" > /dev/null
-        chroot "$BASE_DIR" dpkg-query -f '${binary:Package} ${source:Version}\n' -W > "$BASE_PKGS_FILE".new
+        debootstrap --include="$BASE_PKG" --variant=minbase "$RELEASEVER" "$TMP_BASE_DIR" >> /var/log/risotto/build_image.log
+        chroot "$TMP_BASE_DIR" dpkg-query -f '${binary:Package} ${source:Version}\n' -W > "$BASE_PKGS_FILE".new
     fi
 }
 function install_base() {
     if [ "$INSTALL_TOOL" = "dnf" ]; then
-        OPT=$(dnf_opt "$BASE_DIR" "$BASE_PKG")
+        OPT=$(dnf_opt "$TMP_BASE_DIR" "$BASE_PKG")
         dnf --assumeyes $OPT
     fi
 }
 function new_package() {
     if [ "$INSTALL_TOOL" = "dnf" ]; then
-        OPT=$(dnf_opt_base "$IMAGE_NAME_RISOTTO_IMAGE_DIR")
-		dnf $OPT update
-        OPT=$(dnf_opt "$IMAGE_NAME_RISOTTO_IMAGE_DIR" "$PKG")
+        OPT=$(dnf_opt_base "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP")
+        dnf --assumeno $OPT update >> /var/log/risotto/build_image.log
+        OPT=$(dnf_opt "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP" "$PKG")
         dnf --assumeno $OPT | grep ^" " > "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs.new
     else
-        chroot "$IMAGE_NAME_RISOTTO_IMAGE_DIR" apt update > /dev/null 2>&1
-        chroot "$IMAGE_NAME_RISOTTO_IMAGE_DIR" apt install --no-install-recommends --yes $PKG -s 2>/dev/null|grep ^"Inst " > "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs.new
+        chroot "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP" apt update >> /var/log/risotto/build_image.log 2>&1
+        chroot "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP" apt install --no-install-recommends --yes $PKG -s 2>/dev/null|grep ^"Inst " > "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs.new
     fi
 }
 function install_pkg() {
     if [ "$INSTALL_TOOL" = "dnf" ]; then
-        OPT=$(dnf_opt "$IMAGE_NAME_RISOTTO_IMAGE_DIR" "$PKG")
+        OPT=$(dnf_opt "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP" "$PKG")
         dnf --assumeyes $OPT
     else
-        chroot "$IMAGE_NAME_RISOTTO_IMAGE_DIR" apt install --no-install-recommends --yes $PKG
+        chroot "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP" bash -c "export DEBIAN_FRONTEND=noninteractive; apt install --no-install-recommends --yes $PKG"
     fi
 }
 
-
-if [ ! -f "$BASE_LOCK" ] || [ ! -f "$BASE_TAR" ]; then
+if [ ! -f "$BASE_LOCK" ] || [ ! -d "$BASE_DIR" ]; then
     echo "  - reinstallation de l'image de base"
-    rm -rf "$BASE_DIR"
     new_package_base
     diff -u "$BASE_PKGS_FILE" "$BASE_PKGS_FILE".new && NEW_BASE=false || NEW_BASE=true
-    if [ ! -f "$BASE_TAR" ] || [ "$NEW_BASE" = true ]; then
+    if [ ! -d "$BASE_DIR" ] || [ "$NEW_BASE" = true ]; then
         mkdir -p "$IMAGE_BASE_RISOTTO_BASE_DIR"
+        rm -rf "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP"
         install_base
-        cd "$IMAGE_BASE_RISOTTO_BASE_DIR"
-        tar cf "$BASE_TAR" "$BASE_NAME"
-        cd - > /dev/null
         if [ -f "$BASE_PKGS_FILE" ]; then
             mv "$BASE_PKGS_FILE" "$BASE_PKGS_FILE".old
         fi
         mv "$BASE_PKGS_FILE".new "$BASE_PKGS_FILE"
-        rm -rf "$IMAGE_BASE_RISOTTO_BASE_DIR"
+        rm -rf "$BASE_DIR"
+        mv "$TMP_BASE_DIR" "$BASE_DIR"
     fi
-    rm -rf "$BASE_DIR"
     touch "$BASE_LOCK"
 fi
 
-tar xf "$BASE_TAR"
-mv "$BASE_NAME" "$IMAGE_NAME_RISOTTO_IMAGE_DIR"
+rm -rf "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP"
+cp --reflink=auto -a "$BASE_DIR/" "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP"
 if [ -n "$COPR" ]; then
     #FIXME signature...
     mkdir -p "$REPO_DIR"
@@ -124,17 +118,18 @@ if [ -n "$COPR" ]; then
     cd - > /dev/null
 fi
 if [ "$FUSION" = true ]; then
-    dnf -y install "https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$RELEASEVER.noarch.rpm" --installroot="$IMAGE_NAME_RISOTTO_IMAGE_DIR" > /dev/null
+    dnf -y install "https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$RELEASEVER.noarch.rpm" --installroot="$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP" >> /var/log/risotto/build_image.log
 fi
 
 # FIXME verifier s'il y a des modifs sur pre/post
 if [ -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".base.pkgs ] && [ -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs ]; then
     echo "  - différence(s) avec les paquets de base"
     diff -u "$IMAGE_NAME_RISOTTO_IMAGE_DIR".base.pkgs "$BASE_PKGS_FILE" && INSTALL=false || INSTALL=true
-    [ ! -f "$IMAGE_NAME_RISOTTO_IMAGE_NAME" ] && INSTALL=true
+    [ ! -d "$IMAGE_NAME_RISOTTO_IMAGE_DIR" ] && INSTALL=true
 else
     INSTALL=true
 fi
+
 new_package
 if [ "$INSTALL" = false ]; then
     echo "  - différence(s) avec les paquets de l'image"
@@ -146,26 +141,12 @@ if [ "$INSTALL" = false ]; then
 fi
 if [ "$INSTALL" = true ]; then
     echo "  - installation"
-    if [ -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR"_"$RELEASEVER".version ]; then
-        VERSION=$(cat "$IMAGE_NAME_RISOTTO_IMAGE_DIR"_"$RELEASEVER".version)
+    if [ -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".version ]; then
+        VERSION=$(cat "$IMAGE_NAME_RISOTTO_IMAGE_DIR".version)
     else
         VERSION=0
     fi
-    mkdir "$RISOTTO_IMAGE_DIR/tmp"
-    ORI_DIR=$PWD
-    cd "$RISOTTO_IMAGE_DIR/tmp"
-    if [ ! "$VERSION" = 0 ] && [ -f "$IMAGE_NAME_RISOTTO_IMAGE_NAME" ]; then
-        tar xf "$IMAGE_NAME_RISOTTO_IMAGE_NAME"
-#        if [ "$INSTALL_TOOL" = "apt" ]; then
-#            chown _apt "$IMAGE_NAME"
-#        fi
-#    else
-#        mkdir "$IMAGE_NAME"
-    fi
-    #cd "$IMAGE_NAME"
     make_changelog "$IMAGE_NAME" "$VERSION" "$OS_NAME" "$RELEASEVER" > "$IMAGE_NAME_RISOTTO_IMAGE_DIR"_"$RELEASEVER"_"$VERSION"_changelog.md
-    cd $ORI_DIR
-    rm -rf "$RISOTTO_IMAGE_DIR/tmp"
     install_pkg
     sleep 2
     
@@ -173,25 +154,20 @@ if [ "$INSTALL" = true ]; then
         . "$script"
     done
     
-    CONTAINER=$IMAGE_NAME make_volatile /etc
+    ROOT=$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP make_volatile /etc
     if [ ! "$?" = 0 ]; then
         echo "make_volatile failed"
         exit 1
     fi
-    cd "$RISOTTO_IMAGE_DIR/$IMAGE_NAME"
-    if [ -f "$IMAGE_NAME_RISOTTO_IMAGE_NAME" ]; then
-        mv -f "$IMAGE_NAME_RISOTTO_IMAGE_NAME" "$IMAGE_NAME_RISOTTO_IMAGE_NAME".old
-    fi
-    tar cf "$IMAGE_NAME_RISOTTO_IMAGE_NAME" .
-    sha256sum "$IMAGE_NAME_RISOTTO_IMAGE_NAME" > "$IMAGE_NAME_RISOTTO_IMAGE_NAME".sha
-    cd - > /dev/null
-    cp -f "$BASE_PKGS_FILE" "$IMAGE_NAME_RISOTTO_IMAGE_DIR".base.pkgs
+    rm -rf "$IMAGE_NAME_RISOTTO_IMAGE_DIR"
+    mv "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP" "$IMAGE_NAME_RISOTTO_IMAGE_DIR"
+    cp --reflink=auto -f "$BASE_PKGS_FILE" "$IMAGE_NAME_RISOTTO_IMAGE_DIR".base.pkgs
     mv -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs.new "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs
     mv -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".md5sum.new "$IMAGE_NAME_RISOTTO_IMAGE_DIR".md5sum
     VERSION=$((VERSION + 1)) 
-    echo "$VERSION" > "$IMAGE_NAME_RISOTTO_IMAGE_DIR"_"$RELEASEVER".version
+    echo "$VERSION" > "$IMAGE_NAME_RISOTTO_IMAGE_DIR".version
 fi
-rm -rf "$IMAGE_NAME_RISOTTO_IMAGE_DIR"
+rm -rf "$IMAGE_NAME_RISOTTO_IMAGE_DIR_TMP"
 
 echo "  => OK"
 exit 0

ansible/sbin/compare_image

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+SRV=$1
+if [ -z "$SRV" ]; then
+    echo "usage: $0 machine"
+    exit 1
+fi
+
+dirname="/var/lib/risotto/templates/$SRV"
+if [ ! -d "$dirname" ]; then
+    echo "cannot find $dirname"
+    echo "usage: $0 machine"
+    exit 1
+fi
+cd $dirname
+find -type f | while read a; do
+    cfile="/var/lib/machines/$SRV/usr/share/factory/$a"
+    if [ -f "$cfile" ]; then
+        diff -u "$cfile" "$a"
+    fi
+done
+cd - > /dev/null

ansible/sbin/make_volatile

@@ -1,9 +1,8 @@
 #!/bin/bash -e
-if [ -z $CONTAINER ]; then
-    echo "PAS DE CONTAINER"
+if [ -z $ROOT]; then
+    echo "PAS DE ROOT"
     exit 1
 fi
-ROOT="/var/lib/risotto/images/$CONTAINER"
 echo "$ROOT"
 DESTDIR="$ROOT/usr/lib/tmpfiles.d"
 CONF_DST="/usr/share/factory"

ansible/sbin/update_images

@@ -8,13 +8,15 @@ IMAGE_BASE_RISOTTO_BASE_DIR="$RISOTTO_IMAGE_DIR/image_bases"
 
 rm -f $IMAGE_BASE_RISOTTO_BASE_DIR*.build
 
-ls /var/lib/risotto/images_files/ | while read image; do
-    if [ -d /var/lib/risotto/images_files/"$image" ]; then
-        echo
-        echo "Install image $image"
-        /usr/local/sbin/build_image "$image"
-    fi
-done
+if [ -z "$1" ]; then
+    ls /var/lib/risotto/images_files/ | while read image; do
+        if [ -d /var/lib/risotto/images_files/"$image" ]; then
+            echo
+            echo "Install image $image"
+            /usr/local/sbin/build_image "$image" || true
+        fi
+    done
+fi
 #rm -f $IMAGE_BASE_RISOTTO_BASE_DIR*.build
 
 MACHINES=""
@@ -28,7 +30,7 @@ for nspawn in $(ls /etc/systemd/nspawn/*.nspawn); do
     IMAGE_NAME_RISOTTO_IMAGE_NAME=${content##* }
     diff -q "$IMAGE_NAME_RISOTTO_IMAGE_NAME".sha "$SHA_MACHINE" > /dev/null || (
         echo "Reinstall machine $machine"
-        machinectl stop $machine
+        machinectl stop $machine || true
         while true; do
             machinectl status "$machine" > /dev/null 2>&1 || break
             sleep 1

risotto.conf.example

@@ -1,3 +1,4 @@
 [directories]
 dataset = '/home/gnunux/git/risotto/dataset/seed'
 dest = 'installations'
+dest_templates = 'templates'

sbin/risotto_check_certificates

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+from os import walk
+from datetime import datetime
+
+
+week_day = datetime.now().isocalendar().week
+week_cert = f'certificate_{week_day}.crt'
+
+
+for p, d, f in walk('pki/x509'):
+    if not d and not f:
+        print('empty dir, you can remove it: ', p)
+    if not f:
+        continue
+    if f == ['serial_number']:
+        continue
+    if not p.endswith('/ca') and not p.endswith('/server') and not p.endswith('/client'):
+        print('unknown directory: ', p)
+        continue
+    if week_cert in f:
+        continue
+    for ff in f:
+        if ff.startswith('certificate_') and ff.endswith('.crt'):
+            print(f'old certificat in: ', p)
+            break
+    else:
+        print('cannot find certificat in: ', p)

sbin/risotto_display

@@ -0,0 +1,235 @@
+#!/usr/bin/env python3
+
+from asyncio import run
+from tabulate import tabulate
+from argparse import ArgumentParser
+
+from rougail.utils import normalize_family
+from tiramisu.error import PropertiesOptionError
+from risotto.machine import load, remove_cache, ROUGAIL_NAMESPACE
+
+
+HIDE_SECRET = True
+
+
+def list_to_string(lst):
+    if isinstance(lst, list):
+        return "\n".join([str(val) for val in lst])
+    return lst
+
+
+async def get_files_subelements(type_name, element, files_subelement, files_cols):
+    data = {}
+    if not await element.option('activate').value.get():
+        return data
+    for subelement in files_subelement.values():
+        if subelement['type'] == 'subelement':
+            try:
+                value = list_to_string(await element.option(subelement['key']).value.get())
+            # FIXME except AttributeError:
+            except Exception:
+                value = ''
+        elif subelement['type'] == 'information':
+            value = await element.information.get(subelement['key'], '')
+        elif subelement['type'] == 'none':
+            value = subelement['value']
+        else:
+            raise Exception('unknown subelement')
+        if value != '':
+            files_cols.add(subelement['key'])
+        data[subelement['key']] = value
+    if type_name == 'overrides':
+        data['name'] = f'/systemd/system/{data["source"]}.d/rougail.conf'
+        if not data['engine']:
+            data['engine'] = 'none'
+    elif not data['engine']:
+        data['engine'] = 'cheetah'
+    return data
+
+
+async def services(config, values):
+    files_subelement = {'Source': {'key': 'source', 'type': 'information'},
+                        'Nom': {'key': 'name', 'type': 'subelement'},
+                        'Variable': {'key': 'variable', 'type': 'subelement'},
+                        'Propriétaire': {'key': 'owner', 'type': 'subelement'},
+                        'Groupe': {'key': 'group', 'type': 'subelement'},
+                        'Mode': {'key': 'mode', 'type': 'subelement'},
+                        'Moteur': {'key': 'engine', 'type': 'information'},
+                        }
+    disabled_services = []
+    for service in await config.option.list(type="all"):
+        doc = await service.option.doc()
+        files_lst = []
+        files_cols = set()
+        if not await service.option('manage').value.get():
+            doc += " - unmanaged"
+        if not await service.option('activate').value.get():
+            disabled_services.append([doc])
+        else:
+            for type in await service.list(type="all"):
+                type_name = await type.option.doc()
+                if type_name in ['files', 'overrides']:
+                    for element in await type.list(type="all"):
+                        data = await get_files_subelements(type_name, element, files_subelement, files_cols)
+                        if data:
+                            files_lst.append(data)
+                elif type_name == 'manage':
+                    pass
+                elif type_name == 'activate':
+                    if not await type.value.get():
+                        doc += " - unactivated"
+                else:
+                    print("FIXME " + type_name)
+            if files_lst:
+                keys = [key for key, val in files_subelement.items() if val['key'] in files_cols]
+                values[doc] = {'keys': keys, 'lst': []}
+                for lst in files_lst:
+                    values[doc]['lst'].append([val for key, val in lst.items() if key in files_cols])
+    if disabled_services:
+        values["Services désactivés"] = {'keys': ['Nom'], 'lst': disabled_services}
+
+
+async def table_leader(config, read_only):
+    keys = ['Description']
+    if read_only:
+        keys.append('Cachée')
+    leadership_lst = await config.list(type="all")
+    leader = leadership_lst.pop(0)
+    leader_owner = await leader.owner.get()
+    follower_names = [await follower.option.name() for follower in leadership_lst]
+    doc = await leader.option.doc()
+    properties = await leader.property.get()
+    if 'mandatory' in properties:
+        doc += '*'
+    name = await leader.option.name()
+    lst = [[f'{doc} ({name})']]
+    if read_only:
+        if 'hidden' in properties:
+            hidden = 'oui'
+        else:
+            hidden = ''
+        lst[0].append(hidden)
+    for idx, leader_value in enumerate(await leader.value.get()):
+        keys.append(f'Valeur {idx}')
+        keys.append(f'Utilisateur {idx}')
+        lst[0].append(leader_value)
+        lst[0].append(leader_owner)
+        for follower_idx, follower_name in enumerate(follower_names):
+            follower_option = config.option(follower_name, idx)
+            if idx == 0:
+                doc = await follower_option.option.doc()
+                properties = await follower_option.property.get()
+                if 'mandatory' in properties:
+                    doc += '*'
+                name = await follower_option.option.name()
+                lst.append([f'{doc} ({name})'])
+                if read_only:
+                    if 'hidden' in properties:
+                        hidden = 'oui'
+                    else:
+                        hidden = ''
+                    lst[-1].append(hidden)
+            try:
+                lst[follower_idx + 1].append(list_to_string(await follower_option.value.get()))
+                lst[follower_idx + 1].append(await follower_option.owner.get())
+            except PropertiesOptionError:
+                pass
+#    leader = next leader_iter
+#        if master_values is None:
+#            master_values = await subconfig.value.get()
+    return {'keys': keys, 'lst': lst}
+
+
+async def table(config, prefix_len, values, read_only):
+    lst = []
+    for subconfig in await config.option.list(type="all"):
+#        prefix = prefix_len * 2 * ' '
+#        if await subconfig.option.isoptiondescription():
+#            prefix += '=>'
+#        else:
+#            prefix += '-'
+#        display_str = f'{prefix} {description}'
+#        if name != description:
+#            display_str = f'{display_str} ({name})'
+        name = await subconfig.option.name()
+        doc = await subconfig.option.doc()
+        if prefix_len == 0 and ROUGAIL_NAMESPACE != name:
+            doc = doc.capitalize()
+        if prefix_len == 0 and name == 'services':
+            values['Services'] = {}
+            await services(subconfig, values['Services'])
+        elif await subconfig.option.isoptiondescription():
+            od_name = f'{doc} ({(await subconfig.option.path()).split(".", 1)[1]})'
+            values[od_name] = None
+            if await subconfig.option.isleadership():
+                values[od_name] = await table_leader(subconfig, read_only)
+            else:
+                values[od_name] = await table(subconfig, prefix_len + 1, values, read_only)
+        else:
+            value = list_to_string(await subconfig.value.get())
+            doc = await subconfig.option.doc()
+            properties = await subconfig.property.get()
+            if 'mandatory' in properties:
+                doc += '*'
+            name = await subconfig.option.name()
+            lst.append([f'{doc} ({name})', value])
+            if read_only:
+                if 'hidden' in properties:
+                    hidden = 'oui'
+                else:
+                    hidden = ''
+                lst[-1].append(hidden)
+            lst[-1].append(await subconfig.owner.get())
+    keys = ['Description', 'Valeur']
+    if read_only:
+        keys.append('Cachée')
+    keys.append('Utilisateur')
+    return {'keys': keys, 'lst': lst}
+
+
+async def main():
+    parser = ArgumentParser()
+    parser.add_argument('server_name')
+    parser.add_argument('--read_only', action='store_true')
+    parser.add_argument('--nocache', action='store_true')
+    args = parser.parse_args()
+    if args.nocache:
+        remove_cache()
+
+    values = {}
+    server_name = args.server_name
+    config = await load(hide_secret=HIDE_SECRET,
+                        original_display_name=True,
+                        valid_mandatories=args.read_only,
+                        )
+    if not args.read_only:
+        await config.property.read_write()
+    root_option = config.option(normalize_family(server_name))
+    try:
+        await root_option.option.get()
+    except AttributeError:
+        exit(f'Unable to find {server_name} configuration: {[await o.option.name() for o in await config.option.list(type="optiondescription")]}')
+    await table(root_option, 0, values, args.read_only)
+    for title, dico in values.items():
+        if title == 'Services':
+            if not dico:
+                continue
+            print()
+            print(title)
+            print('=' * len(title))
+            print()
+            for subtitle, dic in dico.items():
+                print()
+                print('   ' + subtitle)
+                print('   ' + '-' * len(subtitle))
+                print()
+                print(tabulate(dic['lst'], headers=dic['keys'], tablefmt="fancy_grid"))
+        elif dico['lst']:
+            print()
+            print(title)
+            print('=' * len(title))
+            print()
+            print(tabulate(dico['lst'], headers=dico['keys'], tablefmt="fancy_grid"))
+
+
+run(main())

sbin/risotto_templates

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+
+from asyncio import run
+from argparse import ArgumentParser
+from traceback import print_exc
+
+from risotto.machine import templates, remove_cache, load, INSTALL_DIR
+
+
+async def main():
+    parser = ArgumentParser()
+    parser.add_argument('server_name')
+    parser.add_argument('--nocache', action='store_true')
+    parser.add_argument('--debug', action='store_true')
+    args = parser.parse_args()
+    if args.nocache:
+        remove_cache()
+
+    config = await load()
+    try:
+        await templates(args.server_name,
+                        config,
+                        )
+    except Exception as err:
+        if args.debug:
+            print_exc()
+        exit(err)
+    print(f'templates generated in {INSTALL_DIR} directory')
+
+
+run(main())

src/risotto/image.py

@@ -1,64 +1,210 @@
-from shutil import copy2, copytree, rmtree
+from shutil import copy2, copytree
 from os import listdir, makedirs
 from os.path import join, isdir, isfile, dirname
 from yaml import load as yaml_load, SafeLoader
-from json import load as json_load
 #
-from rougail import RougailConfig  # , RougailConvert
-#
-from .utils import RISOTTO_CONFIG, SERVERS, MULTI_FUNCTIONS
-
-
-FUNCTIONS_FILE = 'funcs.py'
+from .utils import RISOTTO_CONFIG
 
 
 class ModuleCfg():
     def __init__(self, module_name):
         self.module_name = module_name
         self.dictionaries_dir = []
-        self.modules = []
-        self.functions_file = [FUNCTIONS_FILE]
+        self.functions_file = []
         self.templates_dir = []
+        self.patches_dir = []
         self.extra_dictionaries = {}
         self.servers = []
         self.depends = []
+        self.manuals = []
+        self.tests = []
+        self.providers = []
+        self.suppliers = []
 
     def __repr__(self):
         return str(vars(self))
 
 
-def list_applications() -> dict:
+class Applications:
+    def __init__(self) -> None:
+        self.datasets = RISOTTO_CONFIG.get('directories', {}).get('datasets', ['dataset'])
+        self.application_directories = self._load_application_directories()
+
+    def _load_application_directories(self) -> dict:
+        """List all service applications in datasets
+        Returns something link:
+        {<applicationservice>: seed/<applicationservice>}
+        """
+        applications = {}
+        for dataset_directory in self.datasets:
+            for applicationservice in listdir(dataset_directory):
+                applicationservice_dir = join(dataset_directory, applicationservice)
+                if not isdir(applicationservice_dir) or \
+                        not isfile(join(applicationservice_dir, 'applicationservice.yml')):
+                    continue
+                if applicationservice in applications:
+                    raise Exception(f'multi applicationservice: {applicationservice} ({applicationservice_dir} <=> {applications[applicationservice]})')
+                applications[applicationservice] = applicationservice_dir
+        return applications
+
+
+class Modules:
+    """Modules are defined by the end user
+    A module is the a list of service applications
+    The class collects all the useful information for the module
     """
-    {<applicationservice>: seed/<applicationservice>
-    """
-    dataset_directory = RISOTTO_CONFIG['directories']['dataset']
-    applications = {}
-    for applicationservice in listdir(dataset_directory):
-        applicationservice_dir = join(dataset_directory, applicationservice)
-        if not isdir(applicationservice_dir):
-            continue
-        if applicationservice in applications:
-            raise Exception(f'multi applicationservice: {applicationservice} ({applicationservice_dir} <=> {applications[applicationservice]})')
-        applications[applicationservice] = applicationservice_dir
-    return applications
+
+    def __init__(self,
+                 host_applicationsservices: list,
+                 applicationservices: Applications,
+                 applicationservice_provider: str,
+                 modules_name: list,
+                 modules: dict,
+                 ) -> None:
+        self.application_directories = applicationservices.application_directories
+        self.module_infos = {}
+        self.module_infos['host'] = self._load_module_informations('host',
+                                                                   host_applicationsservices,
+                                                                   is_host=True,
+                                                                   )
+        for module_name in modules_name:
+            if modules_name == 'host':
+                raise Exception('forbidden module name: "host"')
+            self.module_infos[module_name] = self._load_module_informations(module_name,
+                                                                            [applicationservice_provider] + modules[module_name],
+                                                                            is_host=False,
+                                                                            )
+
+    def get(self,
+            module_name: str,
+            ) -> ModuleCfg:
+        return self.module_infos[module_name]
+
+    def _load_module_informations(self,
+                                  module_name: str,
+                                  applicationservices: list,
+                                  is_host: bool,
+                                  ) -> ModuleCfg:
+        """Create a ModuleCfg object and collect informations
+        A module must depend to an unique distribution
+        """
+        cfg = ModuleCfg(module_name)
+        distribution = None
+
+        for applicationservice in applicationservices:
+            ret = self._load_applicationservice(applicationservice,
+                                                cfg,
+                                                )
+            if ret:
+                if distribution:
+                    raise Exception(f'duplicate distribution for {cfg.module_name}: {distribution} and {ret} (dependencies: {cfg.depends}) ')
+                distribution = ret
+        if not is_host and not distribution:
+            raise Exception(f'cannot found any linux distribution for {module_name}')
+        return cfg
+
+    def _load_applicationservice(self,
+                                 appname: str,
+                                 cfg: ModuleCfg,
+                                 ) -> str:
+        """extract informations from an application service and load it's dependency
+        informations collected is store to the module
+
+        returns the name of current distribution, if found
+        """
+        if appname not in self.application_directories:
+            raise Exception(f'cannot find application dependency "{appname}"')
+        cfg.depends.append(appname)
+        as_dir = self.application_directories[appname]
+        self._load_applicationservice_directories(as_dir,
+                                                  cfg,
+                                                  )
+        with open(join(as_dir, 'applicationservice.yml')) as yaml:
+            app = yaml_load(yaml, Loader=SafeLoader)
+        provider = app.get('provider')
+        if provider:
+            cfg.providers.setdefault(provider, [])
+            if appname not in cfg.providers[provider]:
+                cfg.providers[provider].append(appname)
+        supplier = app.get('supplier')
+        if supplier:
+            self.suppliers.setdefault(supplier, [])
+            if appname not in self.suppliers[supplier]:
+                self.suppliers[supplier].append(appname)
+        if 'distribution' in app and app['distribution']:
+            distribution = appname
+        else:
+            distribution = None
+        for depend in app.get('depends', []):
+            if depend in cfg.depends:
+                #this dependancy is already loaded for this module
+                continue
+            ret = self._load_applicationservice(depend,
+                                                cfg,
+                                                )
+            if ret:
+                if distribution:
+                    raise Exception(f'duplicate distribution for {cfg.module_name}: {distribution} and {ret} (dependencies: {cfg.depends}) ')
+                distribution = ret
+        return distribution
+
+    def _load_applicationservice_directories(self,
+                                             as_dir: str,
+                                             cfg: ModuleCfg,
+                                             ) -> None:
+        # dictionaries
+        dictionaries_dir = join(as_dir, 'dictionaries')
+        if isdir(dictionaries_dir):
+            cfg.dictionaries_dir.append(dictionaries_dir)
+        # funcs
+        funcs_dir = join(as_dir, 'funcs')
+        if isdir(funcs_dir):
+            for f in listdir(funcs_dir):
+                if f.startswith('__'):
+                    continue
+                cfg.functions_file.append(join(funcs_dir, f))
+        # templates
+        templates_dir = join(as_dir, 'templates')
+        if isdir(templates_dir):
+            cfg.templates_dir.append(templates_dir)
+        # patches
+        patches_dir = join(as_dir, 'patches')
+        if isdir(patches_dir):
+            cfg.patches_dir.append(patches_dir)
+        # extras
+        extras_dir = join(as_dir, 'extras')
+        if isdir(extras_dir):
+            for extra in listdir(extras_dir):
+                extra_dir = join(extras_dir, extra)
+                if isdir(extra_dir):
+                    cfg.extra_dictionaries.setdefault(extra, []).append(extra_dir)
+        # manual
+        for type in ['image', 'install']:
+            manual_dir = join(as_dir, 'manual')
+            if isdir(join(manual_dir, type)):
+                cfg.manuals.append(manual_dir)
+                break
+        # tests
+        tests_dir = join(as_dir, 'tests')
+        if isdir(tests_dir):
+            cfg.tests.append(tests_dir)
 
 
 def applicationservice_copy(src_file: str,
                             dst_file: str,
-                            copy_if_not_exists: bool,
                             ) -> None:
     if isdir(src_file):
         if not isdir(dst_file):
             makedirs(dst_file)
         for subfilename in listdir(src_file):
-            if not copy_if_not_exists or not isfile(dst_file):
-                src = join(src_file, subfilename)
-                dst = join(dst_file, subfilename)
-                if isfile(src):
-                    copy2(src, dst)
-                else:
-                    copytree(src, dst)
-    elif not copy_if_not_exists or not isfile(dst_file):
+            #if not copy_if_not_exists or not isfile(dst_file):
+            src = join(src_file, subfilename)
+            dst = join(dst_file, subfilename)
+            if isfile(src):
+                copy2(src, dst)
+            else:
+                copytree(src, dst)
+    else:
         dst = dirname(dst_file)
         if not isdir(dst):
             makedirs(dst)
@@ -68,154 +214,6 @@ def applicationservice_copy(src_file: str,
             copytree(src_file, dst_file)
 
 
-def load_applicationservice_cfg(appname: str,
-                                as_dir: str,
-                                install_dir: str,
-                                cfg: ModuleCfg,
-                                copy_manual_dir: bool,
-                                copy_tests: bool,
-                                ) -> None:
-    cfg.modules.append(appname)
-    # dictionaries
-    dictionaries_dir = join(as_dir, 'dictionaries')
-    if isdir(dictionaries_dir):
-        cfg.dictionaries_dir.append(dictionaries_dir)
-    # funcs
-    funcs_dir = join(as_dir, 'funcs')
-    if isdir(funcs_dir):
-        for f in listdir(funcs_dir):
-            if f.startswith('__'):
-                continue
-            cfg.functions_file.append(join(funcs_dir, f))
-    # templates
-    templates_dir = join(as_dir, 'templates')
-    if isdir(templates_dir):
-        cfg.templates_dir.append(templates_dir)
-    # extras
-    extras_dir = join(as_dir, 'extras')
-    if isdir(extras_dir):
-        for extra in listdir(extras_dir):
-            extra_dir = join(extras_dir, extra)
-            if isdir(extra_dir):
-                cfg.extra_dictionaries.setdefault(extra, []).append(extra_dir)
-    if copy_manual_dir:
-        # manual
-        for type in ['image', 'install']:
-            manual_dir = join(as_dir, 'manual', type)
-            if not isdir(manual_dir):
-                continue
-            for filename in listdir(manual_dir):
-                src_file = join(manual_dir, filename)
-                if type == 'image':
-                    dst_file = join(install_dir, 'manual', filename)
-                    copy_if_not_exists = False
-                else:
-                    dst_file= join(install_dir, '..', filename)
-                    copy_if_not_exists = True
-                applicationservice_copy(src_file,
-                                        dst_file,
-                                        copy_if_not_exists,
-                                        )
-    if copy_tests:
-        tests_dir = join(as_dir, 'tests')
-        if isdir(tests_dir):
-            for filename in listdir(tests_dir):
-                src_file = join(tests_dir, filename)
-                dst_file = join(install_dir, 'tests', filename)
-                applicationservice_copy(src_file,
-                                        dst_file,
-                                        False,
-                                        )
-
-
-def load_applicationservice(appname: str,
-                            install_dir: str,
-                            cfg: ModuleCfg,
-                            applications: dict,
-                            copy_manual_dir: bool,
-                            copy_tests: bool,
-                            providers: dict,
-                            suppliers: dict,
-                            ) -> None:
-    if appname not in applications:
-        raise Exception(f'cannot find application dependency "{appname}"')
-    as_dir = applications[appname]
-    applicationservice_file = join(as_dir, 'applicationservice.yml')
-    if not isfile(applicationservice_file):
-        raise Exception(f'cannot find application service file "{applicationservice_file}"')
-    load_applicationservice_cfg(appname,
-                                as_dir,
-                                install_dir,
-                                cfg,
-                                copy_manual_dir,
-                                copy_tests,
-                                )
-    cfg.depends.append(appname)
-    with open(applicationservice_file) as yaml:
-        app = yaml_load(yaml, Loader=SafeLoader)
-    provider = app.get('provider')
-    if provider:
-        providers.setdefault(provider, [])
-        if appname not in providers[provider]:
-            providers[provider].append(appname)
-    supplier = app.get('supplier')
-    if supplier:
-        suppliers.setdefault(supplier, [])
-        if appname not in suppliers[supplier]:
-            suppliers[supplier].append(appname)
-    if 'distribution' in app and app['distribution']:
-        distribution = appname
-    else:
-        distribution = None
-    for xml in app.get('depends', []):
-        if xml in cfg.depends:
-            continue
-        ret = load_applicationservice(xml,
-                                      install_dir,
-                                      cfg,
-                                      applications,
-                                      copy_manual_dir,
-                                      copy_tests,
-                                      providers,
-                                      suppliers,
-                                      )
-        if ret:
-            if distribution:
-                raise Exception(f'duplicate distribution for {cfg.module_name}: {distribution} and {ret} (dependencies: {cfg.depends}) ')
-            distribution = ret
-    return distribution
-
-
-def load_image_informations(module_name: str,
-                            install_dir: str,
-                            datas: dict,
-                            applications: dict,
-                            copy_manual_dir: bool,
-                            copy_tests: bool,
-                            providers: dict,
-                            suppliers: dict,
-                            ) -> ModuleCfg:
-    cfg = ModuleCfg(module_name)
-    distribution = None
-    for applicationservice in datas['applicationservices']:
-        ret = load_applicationservice(applicationservice,
-                                      install_dir,
-                                      cfg,
-                                      applications,
-                                      copy_manual_dir,
-                                      copy_tests,
-                                      providers,
-                                      suppliers,
-                                      )
-        if ret:
-            if distribution:
-                raise Exception(f'duplicate distribution for {cfg.module_name}: {distribution} and {ret} (dependencies: {cfg.depends}) ')
-            distribution = ret
-    if module_name != 'host' and not distribution:
-        raise Exception(f'cannot found any linux distribution for {module_name}')
-    return cfg
-
-
 async def valid_mandatories(config):
     mandatories = await config.value.mandatory()
     if mandatories:
@@ -232,60 +230,3 @@ async def valid_mandatories(config):
             # await value_pprint(await config.value.dict(), config)
         exit(1)
         #raise Exception('configuration has mandatories variables without values')
-
-
-def load_config(copy_manual_dir=False,
-                copy_tests=False,
-                clean_directories=False,
-                ):
-    module_infos = {}
-    applications = list_applications()
-    with open('servers.json', 'r') as server_fh:
-        jsonfile = json_load(server_fh)
-        SERVERS.update(jsonfile['servers'])
-        modules = jsonfile['modules']
-    for module_name, datas in modules.items():
-        providers = {}
-        suppliers = {}
-        install_dir = join(RISOTTO_CONFIG['directories']['dest'], module_name)
-        if clean_directories:
-            if isdir(install_dir):
-                rmtree(install_dir)
-            makedirs(install_dir)
-        module_infos[module_name] = {'infos': load_image_informations(module_name,
-                                                                      install_dir,
-                                                                      datas,
-                                                                      applications,
-                                                                      copy_manual_dir,
-                                                                      copy_tests,
-                                                                      providers,
-                                                                      suppliers,
-                                                                      ),
-                                     'providers': providers,
-                                     'suppliers': suppliers,
-                                     'install_dir': install_dir,
-                                     }
-    return module_infos
-#
-#
-def load_module_config(module_name: str,
-                       module_info: dict,
-                       ):
-    cfg = RougailConfig.copy()
-    cfg['variable_namespace'] = ROUGAIL_NAMESPACE
-    cfg['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
-    if module_name == 'host':
-        #FIXME server_name == host ?
-        #FIXME cfg['tmpfile_dest_dir'] = datas['values'][f'{ROUGAIL_NAMESPACE}.host_install_dir'] + '/host/configurations/host'
-        cfg['default_systemd_directory'] = '/usr/local/lib/systemd'
-    cfg['templates_dir'] = module_info['infos'].templates_dir
-    cfg['dictionaries_dir'] = module_info['infos'].dictionaries_dir
-    cfg['functions_file'] = module_info['infos'].functions_file
-    cfg['multi_functions'] = MULTI_FUNCTIONS
-    cfg['extra_dictionaries'] = module_info['infos'].extra_dictionaries
-    cfg['extra_annotators'] = ['risotto.rougail']
-    cfg['internal_functions'] = list(FUNCTIONS.keys())
-    cfg['force_convert_dyn_option_description'] = True
-    cfg['module_name'] = module_name
-    #cfg['patches_dir'] = join(test_dir, 'patches')
-    return cfg

src/risotto/machine.py

@@ -1,16 +1,19 @@
-from .utils import SERVERS, SERVERS_JSON, MULTI_FUNCTIONS, load_domains
-from .image import load_config, valid_mandatories  # , load_modules_rougail_config
-from rougail import RougailConfig, RougailConvert
+from .utils import MULTI_FUNCTIONS, load_zones, value_pprint, RISOTTO_CONFIG
+from .image import Applications, Modules, valid_mandatories, applicationservice_copy
 from .rougail.annotator import calc_providers, calc_providers_global, calc_providers_dynamic, calc_providers_dynamic_follower, calc_providers_follower
-from os.path import isfile
+
+from rougail import RougailConfig, RougailConvert
+from os import remove, makedirs, listdir
+from os.path import isfile, isdir, abspath
 from json import dump as json_dump, load as json_load
+from yaml import load as yaml_load, SafeLoader
 #
-from tiramisu import Config
-from .utils import value_pprint
+from tiramisu import Config, valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal, calc_value
 from rougail.utils import normalize_family
 from rougail import RougailSystemdTemplate
-#
-#
+from shutil import rmtree
+
+
 def tiramisu_display_name(kls,
                           dyn_name: 'Base'=None,
                           suffix: str=None,
@@ -23,49 +26,77 @@ def tiramisu_display_name(kls,
     return name
 
 
-async def set_values(server_name, config, datas):
-    if 'values' not in datas:
-        return
-    server_path = normalize_family(server_name)
-    for vpath, value in datas['values'].items():
-        path = f'{server_path}.{vpath}'
-        try:
-            if isinstance(value, dict):
-                for idx, val in value.items():
-                    await config.option(path, int(idx)).value.set(val)
-            else:
-                await config.option(path).value.set(value)
-        except Exception as err:
-            await value_pprint(await config.value.dict(), config)
-            error_msg = f'cannot configure variable {vpath} for server "{server_name}": {err}'
-            raise Exception(error_msg) from err
-
-
-def get_ip_from_domain(domain):
-    if not domain:
-        return
-    hostname, domainname = domain.split('.', 1)
-    return DOMAINS[domainname][1][DOMAINS[domainname][0].index(hostname)]
-    return optiondescription['option_0']
-
-
+CONFIG_FILE = 'servers.yml'
 ROUGAIL_NAMESPACE = 'general'
 ROUGAIL_NAMESPACE_DESCRIPTION = 'Général'
-FUNCTIONS = {'get_ip_from_domain': get_ip_from_domain,
-             'calc_providers': calc_providers,
+TIRAMISU_CACHE = 'tiramisu_cache.py'
+VALUES_CACHE = 'values_cache.json'
+INFORMATIONS_CACHE = 'informations_cache.json'
+INSTALL_DIR = RISOTTO_CONFIG['directories']['dest']
+INSTALL_TEMPLATES_DIR = RISOTTO_CONFIG['directories']['dest_templates']
+FUNCTIONS = {'calc_providers': calc_providers,
              'calc_providers_global': calc_providers_global,
              'calc_providers_dynamic': calc_providers_dynamic,
              'calc_providers_dynamic_follower': calc_providers_dynamic_follower,
              'calc_providers_follower': calc_providers_follower,
+             'valid_network_netmask': valid_network_netmask,
+             'valid_ip_netmask': valid_ip_netmask,
+             'valid_broadcast': valid_broadcast,
+             'valid_in_network': valid_in_network,
+             'valid_not_equal': valid_not_equal,
+             'calc_value': calc_value,
+             'normalize_family': normalize_family,
              }
 
 
+def re_create(dirname):
+    if isdir(dirname):
+        rmtree(dirname)
+    makedirs(dirname)
+
+
+def remove_cache():
+    if isfile(TIRAMISU_CACHE):
+        remove(TIRAMISU_CACHE)
+    if isfile(VALUES_CACHE):
+        remove(VALUES_CACHE)
+    if isfile(INFORMATIONS_CACHE):
+        remove(INFORMATIONS_CACHE)
+
+
 async def templates(server_name,
                     config,
-                    templates_informations,
                     just_copy=False,
                     ):
-    engine = RougailSystemdTemplate(config, templates_informations)
+    subconfig = config.option(normalize_family(server_name))
+    try:
+        await subconfig.option.get()
+    except:
+        servers = [await server.option.description() for server in await config.option.list('optiondescription')]
+        raise Exception(f'cannot find server name "{server_name}": {servers}')
+
+    rougailconfig = RougailConfig.copy()
+    rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE
+    rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
+    rougailconfig['tmp_dir'] = 'tmp'
+    if not just_copy:
+        rougailconfig['destinations_dir'] = INSTALL_DIR
+    else:
+        rougailconfig['destinations_dir'] = INSTALL_TEMPLATES_DIR
+    rougailconfig['templates_dir'] = await subconfig.information.get('templates_dir')
+    rougailconfig['patches_dir'] = await subconfig.information.get('patches_dir')
+    rougailconfig['functions_file'] = await subconfig.information.get('functions_files')
+    is_host = await subconfig.information.get('module') == 'host'
+    if is_host:
+        host_install_dir = f'{ROUGAIL_NAMESPACE}.host_install_dir'
+        rougailconfig['tmpfile_dest_dir'] = await subconfig.option(host_install_dir).value.get()
+        rougailconfig['default_systemd_directory'] = '/usr/local/lib/systemd'
+    else:
+        rougailconfig['tmpfile_dest_dir'] = '/usr/local/lib'
+        rougailconfig['default_systemd_directory'] = '/systemd'
+    re_create(rougailconfig['destinations_dir'])
+    re_create(rougailconfig['tmp_dir'])
+    engine = RougailSystemdTemplate(subconfig, rougailconfig)
     if just_copy:
         # for all engine to none
         ori_engines = {}
@@ -79,95 +110,211 @@ async def templates(server_name,
     except Exception as err:
         print()
         print(f'=== Configuration: {server_name} ===')
-        values = await config.value.dict()
-        await value_pprint(values, config)
-        print(err)
-        print(await config.option('general.nginx.nginx_default_http').value.get())
+        values = await subconfig.value.dict()
+        await value_pprint(values, subconfig)
         raise err from err
     if just_copy:
         for eng, old_engine in ori_engines.items():
             engine.engines[eng] = old_engine
 
 
-async def load(cache_file,
-               cache_values,
-               clean_directories=False,
-               copy_manual_dir=False,
-               copy_tests=False,
-               hide_secret=False,
-               ):
-    display_name=tiramisu_display_name
-    #load_zones()
-#    # load images
-    #FIXME useful
-    module_infos = load_config(copy_manual_dir, 
-                               copy_tests,
-                               clean_directories,
-                               )
-#    modules_rougail_config = load_modules_rougail_config(module_infos)
-    cfg = RougailConfig.copy()
-    cfg['variable_namespace'] = ROUGAIL_NAMESPACE
-    cfg['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
-    cfg['multi_functions'] = MULTI_FUNCTIONS
-    cfg['extra_annotators'] = ['risotto.rougail']
-    cfg['internal_functions'] = list(FUNCTIONS.keys())
-    cfg['force_convert_dyn_option_description'] = True
-#        cfg['module_name'] = module_name
-    functions_files = set()
-    load_domains()
-    for server_name, datas in SERVERS.items():
-        module_info = module_infos[datas['module']]
-        functions_files |= set(module_info['infos'].functions_file)
-    cfg['functions_file'] = list(functions_files)
-    if not isfile(cache_file):
-        eolobj = RougailConvert(cfg)
+class Loader:
+    def __init__(self,
+                 cache_file,
+                 cache_values,
+                 cache_informations,
+                 clean_directories,
+                 hide_secret,
+                 original_display_name,
+                 valid_mandatories,
+                 config_file=CONFIG_FILE,
+                 ):
+        self.cache_file = cache_file
+        self.cache_values = cache_values
+        self.cache_informations = cache_informations
+        self.hide_secret = hide_secret
+        self.original_display_name = original_display_name
+        self.valid_mandatories = valid_mandatories
+        self.config_file = config_file
+        if clean_directories:
+            if isdir(INSTALL_DIR):
+                rmtree(INSTALL_DIR)
+            makedirs(INSTALL_DIR)
+
+    def before(self):
+        with open(self.config_file, 'r') as server_fh:
+            self.servers_json = yaml_load(server_fh, Loader=SafeLoader)
+        cfg = RougailConfig.copy()
+        cfg['variable_namespace'] = ROUGAIL_NAMESPACE
+        cfg['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
+        cfg['multi_functions'] = MULTI_FUNCTIONS
+        cfg['extra_annotators'] = ['risotto.rougail']
+        cfg['internal_functions'] = list(FUNCTIONS.keys())
+        cfg['force_convert_dyn_option_description'] = True
         cfg['risotto_globals'] = {}
-        for server_name, datas in SERVERS.items():
-            module_info = module_infos[datas['module']]
-            cfg['dictionaries_dir'] = module_info['infos'].dictionaries_dir
-            cfg['extra_dictionaries'] = module_info['infos'].extra_dictionaries
-            informations = SERVERS_JSON['servers'][server_name].get('informations')
-            if informations:
-                cfg['risotto_globals'][server_name] = {'global:server_name': server_name,
-                                                       'global:zones_name': informations['zones_name'],
-                                                       'global:zones_list': list(range(len(informations['zones_name']))),
-                                                       }
-                values = []
-                for s_idx in cfg['risotto_globals'][server_name]['global:zones_list']:
-                    if not s_idx:
-                        values.append(server_name)
-                    else:
-                        values.append(informations['extra_domainnames'][s_idx - 1])
-                cfg['risotto_globals'][server_name]['global:server_names'] = values
-            else:
-                cfg['risotto_globals'][server_name] = {'global:server_name': server_name}
-            cfg['risotto_globals'][server_name]['global:module_name'] = datas['module']
-            eolobj.load_dictionaries(path_prefix=server_name)
-        tiram_obj = eolobj.save(cache_file)
-    else:
-        with open(cache_file) as fh:
-            tiram_obj = fh.read()
-    optiondescription = FUNCTIONS.copy()
-    try:
-        exec(tiram_obj, None, optiondescription)
-    except Exception as err:
-        print(tiram_obj)
-        raise Exception(f'unknown error when load tiramisu object {err}') from err
-    config = await Config(optiondescription['option_0'],
-                          display_name=display_name,
-                          )
-    if not isfile(cache_values):
+
+        rougail = RougailConvert(cfg)
+        self.templates_dir = {}
+        self.patches_dir = {}
+        functions_files = set()
+        self.functions_files = {}
+        applicationservices = Applications()
+        zones = self.servers_json['zones']
+        self.modules = {}
+        for host_name, datas in self.servers_json['hosts'].items():
+            modules_name = {mod_datas['module'] for mod_datas in datas['servers'].values()}
+            modules = Modules(datas['applicationservices'],
+                              applicationservices,
+                              datas['applicationservice_provider'],
+                              modules_name,
+                              self.servers_json['modules']
+                              )
+            module_info = modules.get('host')
+            cfg['risotto_globals'][host_name] = {'global:server_name': host_name,
+                                                 'global:module_name': 'host',
+                                                 'global:host_install_dir': abspath(INSTALL_DIR),
+                                                 }
+            functions_files |= set(module_info.functions_file)
+            self.load_dictionaries(cfg, module_info, host_name, rougail)
+            modules_info = {}
+            for server_name, server_datas in datas['servers'].items():
+                module_info = modules.get(server_datas['module'])
+                zones_name = server_datas['informations']['zones_name']
+                values = [f'{server_name}.{zones[zone_name]["domain_name"]}' for zone_name in zones_name]
+                cfg['risotto_globals'][values[0]] = {'global:host_name': host_name,
+                                                     'global:server_name': values[0],
+                                                     'global:server_names': values,
+                                                     'global:zones_name': zones_name,
+                                                     'global:zones_list': list(range(len(zones_name))),
+                                                     'global:module_name': server_datas['module'],
+                                                     }
+                server_datas['server_name'] = values[0]
+                functions_files |= set(module_info.functions_file)
+                self.load_dictionaries(cfg, module_info, values[0], rougail)
+                modules_info[module_info.module_name] = module_info.depends
+            self.modules[host_name] = modules_info
+        cfg['functions_file'] = list(functions_files)
+        self.tiram_obj = rougail.save(self.cache_file)
+
+    def load_dictionaries(self, cfg, module_info, server_name, rougail):
+        cfg['dictionaries_dir'] = module_info.dictionaries_dir
+        cfg['extra_dictionaries'] = module_info.extra_dictionaries
+        cfg['functions_file'] = module_info.functions_file
+        rougail.load_dictionaries(path_prefix=server_name)
+        self.templates_dir[server_name] = module_info.templates_dir
+        self.patches_dir[server_name] = module_info.patches_dir
+        self.functions_files[server_name] = module_info.functions_file
+
+    async def load(self):
+        optiondescription = FUNCTIONS.copy()
+        try:
+            exec(self.tiram_obj, None, optiondescription)
+        except Exception as err:
+            print(self.tiram_obj)
+            raise Exception(f'unknown error when load tiramisu object {err}') from err
+        if self.original_display_name:
+            display_name = None
+        else:
+            display_name = tiramisu_display_name
+        self.config = await Config(optiondescription['option_0'],
+                                   display_name=display_name,
+                                   )
+
+    async def after(self):
+        config = self.config
         await config.property.pop('validator')
         await config.property.pop('cache')
-        for server_name, datas in SERVERS.items():
-            await set_values(server_name, config, datas)
+        load_zones(self.servers_json)
+        await config.information.set('zones', self.servers_json['zones'])
+        for host_name, hosts_datas in self.servers_json['hosts'].items():
+            information = config.option(normalize_family(host_name)).information
+            await information.set('module', 'host')
+            await information.set('templates_dir', self.templates_dir[host_name])
+            await information.set('patches_dir', self.patches_dir[host_name])
+            await information.set('functions_files', self.functions_files[host_name])
+            await self.set_values(host_name, config, hosts_datas)
+            for datas in hosts_datas['servers'].values():
+                server_name = datas['server_name']
+                information = config.option(normalize_family(server_name)).information
+                await information.set('module', datas['module'])
+                await information.set('templates_dir', self.templates_dir[server_name])
+                await information.set('patches_dir', self.patches_dir[server_name])
+                await information.set('functions_files', self.functions_files[server_name])
+                await self.set_values(server_name, config, datas)
+
+        # FIXME only one host_name is supported
+        await config.information.set('modules', self.modules[host_name])
+        # await config.information.set('modules', {module_name: module_info.depends for module_name, module_info in self.module_infos.items() if module_name in modules})
         await config.property.read_only()
         await config.property.add('cache')
-        await valid_mandatories(config)
-        with open(cache_values, 'w') as fh:
+        if self.valid_mandatories:
+            await valid_mandatories(config)
+        with open(self.cache_values, 'w') as fh:
             json_dump(await config.value.exportation(), fh)
+        with open(self.cache_informations, 'w') as fh:
+            json_dump(await config.information.exportation(), fh)
+
+    async def set_values(self,
+                         server_name,
+                         config,
+                         datas,
+                         ):
+        if 'values' not in datas:
+            return
+        server_path = normalize_family(server_name)
+        await config.owner.set(self.config_file)
+        for vpath, value in datas['values'].items():
+            path = f'{server_path}.{vpath}'
+            try:
+                if isinstance(value, dict):
+                    for idx, val in value.items():
+                        await config.option(path, int(idx)).value.set(val)
+                else:
+                    await config.option(path).value.set(value)
+            except Exception as err:
+                await value_pprint(await config.value.dict(), config)
+                error_msg = f'cannot configure variable {vpath} for server "{server_name}": {err}'
+                raise Exception(error_msg) from err
+        await config.owner.set('user')
+
+    async def finish(self):
+        await self.config.property.read_only()
+
+
+class LoaderCache(Loader):
+    def before(self):
+        with open(self.cache_file) as fh:
+            self.tiram_obj = fh.read()
+
+    async def after(self):
+        with open(self.cache_values, 'r') as fh:
+            await self.config.value.importation(json_load(fh))
+        with open(self.cache_informations, 'r') as fh:
+            informations = json_load(fh)
+            # null is not a valid key in json => 'null'
+            informations[None] = informations.pop('null')
+            await self.config.information.importation(informations)
+
+
+async def load(clean_directories=False,
+               hide_secret=False,
+               original_display_name: bool=False,
+               valid_mandatories: bool=True,
+               ):
+    if isfile(TIRAMISU_CACHE) and isfile(VALUES_CACHE) and isfile(INFORMATIONS_CACHE):
+        loader_obj = LoaderCache
     else:
-        with open(cache_values, 'r') as fh:
-            await config.value.importation(json_load(fh))
-        await config.property.read_only()
-    return module_infos, cfg, config
+        loader_obj = Loader
+    loader = loader_obj(TIRAMISU_CACHE,
+                        VALUES_CACHE,
+                        INFORMATIONS_CACHE,
+                        clean_directories,
+                        hide_secret,
+                        original_display_name,
+                        valid_mandatories,
+                        )
+    loader.before()
+    await loader.load()
+    await loader.after()
+    await loader.finish()
+    return loader.config

src/risotto/rougail/annotator.py

@@ -1,10 +1,12 @@
 from rougail.annotator.variable import Walk
+from rougail.error import DictConsistencyError
 from risotto.utils import _, multi_function
 from warnings import warn
 
 
-
 def _parse_kwargs(provider, dns, kwargs, index=None):
+    if not isinstance(dns, list):
+        raise Exception('pfff')
     values = {}
     for key, value in kwargs.items():
         if '_' not in key:
@@ -14,7 +16,15 @@ def _parse_kwargs(provider, dns, kwargs, index=None):
     for idx, data in values.items():
         if index is not None and int(idx) != index:
             continue
-        if 'dns' not in data or (isinstance(data['dns'], list) and dns not in data['dns']) or (not isinstance(data['dns'], list) and data['dns'] != dns):
+        if 'dns' not in data:
+            continue
+        if isinstance(data['dns'], list):
+            for ddns in data['dns']:
+                if ddns in dns:
+                    break
+            else:
+                continue
+        elif data['dns'] not in dns:
             continue
         del data['dns']
         yield data
@@ -142,7 +152,13 @@ class Annotator(Walk):
                 continue
             nf_dns = variable.path.split('.', 1)[0]
             server_name = self.objectspace.space.variables[nf_dns].doc
-            self.suppliers.setdefault(variable.supplier, []).append({'option': variable, 'dns': server_name, 'path_prefix': nf_dns, 'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'])})
+            self.suppliers.setdefault(variable.supplier, []).append({'option': variable,
+                                                                     'dns': server_name,
+                                                                     'path_prefix': nf_dns,
+                                                                     'server_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names'],
+                                                                     'zone_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'],
+                                                                     'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'])
+                                                                     })
 
     def convert_suppliers(self):
         for supplier, data in self.suppliers.items():
@@ -152,10 +168,16 @@ class Annotator(Walk):
                 if supplier not in self.providers:
                     continue
                 for p_dico in self.providers[supplier]:
-                    if s_dico['zones'] & p_dico['zones']:
-                        s_dico['option'].value = p_dico['dns']
+                    common_zones = s_dico['zones'] & p_dico['zones']
+                    if common_zones:
+                        for idx, zone in enumerate(p_dico['zone_names']):
+                            if zone in common_zones:
+                                break
+                        dns = p_dico['server_names'][idx]
+#                        dns = p_dico["dns"]
+                        s_dico['option'].value = dns
                         new_value = self.objectspace.value(None)
-                        new_value.name = p_dico['dns']
+                        new_value.name = dns
                         s_dico['option'].value = [new_value]
                         break
 
@@ -168,6 +190,10 @@ class Annotator(Walk):
             nf_dns = variable.path.split('.', 1)[0]
             server_name = self.objectspace.space.variables[nf_dns].doc
             provider_name = variable.provider
+            if self.objectspace.rougailconfig['risotto_globals'][server_name]['global:module_name'] == 'host':
+                server_names = [server_name]
+            else:
+                server_names = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names']
             if ':' in provider_name:
                 key_name, key_type = provider_name.rsplit(':', 1)
                 is_provider = False
@@ -175,7 +201,13 @@ class Annotator(Walk):
                 key_name = key_type = provider_name
                 is_provider = True
                 if provider_name != 'Host':
-                    self.providers.setdefault(provider_name, []).append({'option': variable, 'dns': server_name, 'path_prefix': nf_dns, 'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'])})
+                    self.providers.setdefault(provider_name, []).append({'option': variable,
+                                                                         'dns': server_name,
+                                                                         'path_prefix': nf_dns,
+                                                                         'server_names': server_names,
+                                                                         'zone_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'],
+                                                                         'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name']),
+                                                                         })
             if key_name != 'global' and key_name not in self.suppliers:
                 #warn(f'cannot find supplier "{key_name}" for "{server_name}"')
                 continue
@@ -234,10 +266,12 @@ class Annotator(Walk):
             if key_name != 'global':
                 param = self.objectspace.param(variable.xmlfiles)
                 param.name = 'dns'
-                param.text = server_name
+                param.text = server_names
                 fill.param.append(param)
             if key_name == 'global':
                 param = self.objectspace.param(variable.xmlfiles)
+                if provider_name not in self.objectspace.rougailconfig['risotto_globals'][server_name]:
+                    raise DictConsistencyError(f'cannot find provider "{provider_name}" for variable "{variable.name}"', 200, variable.xmlfiles)
                 param.text = self.objectspace.rougailconfig['risotto_globals'][server_name][provider_name]
                 param.name = 'value'
                 fill.param.append(param)
@@ -296,48 +330,3 @@ class Annotator(Walk):
             if not hasattr(self.objectspace.space.variables[nf_dns].constraints, 'fill'):
                 self.objectspace.space.variables[nf_dns].constraints.fill = []
             self.objectspace.space.variables[nf_dns].constraints.fill.append(fill)
-
-#    def convert_get_linked_information(self):
-#        if not hasattr(self.objectspace.space, 'constraints') or \
-#                not hasattr(self.objectspace.space.constraints, 'fill'):
-#            return
-#        for fill in self.objectspace.space.constraints.fill:
-#            if fill.name == 'get_linked_configuration':
-#                # add server_name
-#                param = self.objectspace.param(fill.xmlfiles)
-#                param.name = 'server_name'
-#                param.type = 'information'
-#                param.text = 'server_name'
-#                fill.param.append(param)
-#                # add current_user
-#                param = self.objectspace.param(fill.xmlfiles)
-#                param.name = 'current_user'
-#                param.type = 'information'
-#                param.text = 'current_user'
-#                fill.param.append(param)
-#                # add test
-#                param = self.objectspace.param(fill.xmlfiles)
-#                param.name = 'test'
-#                param.type = 'target_information'
-#                param.text = 'test'
-#                fill.param.append(param)
-#
-#    def convert_provider(self):
-#        if not hasattr(self.objectspace.space, 'variables'):
-#            return
-#        for family in self.get_families():
-#            if not hasattr(family, 'provider'):
-#                continue
-#            if 'dynamic' not in vars(family):
-#                raise Exception(_(f'{family.name} is not a dynamic family so cannot have provider attribute'))
-#            if not hasattr(family, 'information'):
-#                family.information = self.objectspace.information(family.xmlfiles)
-#            family.information.provider = family.provider
-#            del family.provider
-#        for variable in self.get_variables():
-#            if not hasattr(variable, 'provider'):
-#                continue
-#            if not hasattr(variable, 'information'):
-#                variable.information = self.objectspace.information(variable.xmlfiles)
-#            variable.information.provider = variable.provider
-#            del variable.provider

src/risotto/utils.py

@@ -1,18 +1,11 @@
 from os import  environ
-from json import load
 from typing import List
 from ipaddress import ip_address
 from toml import load as toml_load
 from pprint import pprint
 
 
-SETTINGS = {'config': None}
 MULTI_FUNCTIONS = []
-DOMAINS = {}
-ZONES = {}
-SERVERS_JSON = {}
-SERVERS = {}
-CONFIGS = {}
 
 
 with open(environ.get('CONFIG_FILE', 'risotto.conf'), 'r') as fh:
@@ -40,68 +33,20 @@ async def value_pprint(dico, config):
     pprint(pprint_dict)
 
 
-def load_zones_server():
-    if 'zones' in SERVERS_JSON:
-        return
-    with open('servers.json', 'r') as server_fh:
-        SERVERS_JSON.update(load(server_fh))
-
-
-def load_zones():
-    global ZONES
-    if ZONES:
-        return
-
-    load_zones_server()
-    ZONES.update(SERVERS_JSON['zones'])
-    for server_name, server in SERVERS_JSON['servers'].items():
-        if 'informations' not in server:
-            continue
-        server_zones = server['informations']['zones_name']
-        server_extra_domainnames = server['informations'].get('extra_domainnames', [])
-        if len(server_zones) > 1 and len(server_zones) != len(server_extra_domainnames) + 1:
-            raise Exception(f'the server "{server_name}" has more that one zone, please set correct number of extra_domainnames ({len(server_zones) - 1} instead of {len(server_extra_domainnames)})')
-
-        for idx, zone_name in enumerate(server_zones):
-            zone_domain_name = ZONES[zone_name]['domain_name']
-            if idx == 0:
-                zone_server_name = server_name
-            else:
-                zone_server_name = server_extra_domainnames[idx - 1]
-            server_domain_name = zone_server_name.split('.', 1)[1]
-            if zone_domain_name and zone_domain_name != server_domain_name:
-                raise Exception(f'wrong server_name "{zone_server_name}" in zone "{zone_name}" should ends with "{zone_domain_name}"')
-            ZONES[zone_name].setdefault('hosts', []).append(server_name)
-
-
-def load_domains():
-    global DOMAINS
-    if DOMAINS:
-        return
-    load_zones()
-    for zone_name, zone in SERVERS_JSON['zones'].items():
-        if 'domain_name' in zone:
-            hosts = []
-            ips = []
-            for host in ZONES[zone_name].get('hosts', []):
-                hosts.append(host.split('.', 1)[0])
-                ips.append(_get_ip(host, [zone_name], 0))
-            DOMAINS[zone['domain_name']] = (tuple(hosts), tuple(ips))
+def load_zones(servers_json):
+    zones = servers_json['zones']
+    for host_name, hosts in servers_json['hosts'].items():
+        for server_name, server in hosts['servers'].items():
+            server_zones = server['informations']['zones_name']
+            for idx, zone_name in enumerate(server_zones):
+                zone = zones[zone_name]
+                zone.setdefault('hosts', {})
+                zone['hosts'][server_name] = _get_ip(server_name, zone)
 
 
 def _get_ip(server_name: str,
-            zones_name: List[str],
-            index: str,
+            zone: dict,
             ) -> str:
-    if server_name is None or zones_name is None:
-        return
-    load_zones()
-    index = int(index)
-    zone_name = zones_name[index]
-    if zone_name not in ZONES:
-        raise ValueError(f"cannot set IP in unknown zone '{zone_name}'")
-    zone = ZONES[zone_name]
-    if server_name not in zone['hosts']:
-        raise ValueError(f"cannot set IP in unknown server '{server_name}'")
-    server_index = zone['hosts'].index(server_name)
+    # FIXME make a cache, machine should not change IP
+    server_index = len(zone['hosts'])
     return str(ip_address(zone['start_ip']) + server_index)