fix(src/risotto/rougail/annotator.py): do not raise if suppliers not exists

close #9

ansible/host.yml

@@ -10,12 +10,40 @@
   ansible.builtin.hostname:
     name: "{{ inventory_hostname }}"
 
-- name: "Packages installation"
+- name: "Install packages"
   apt:
     pkg: "{{ vars[inventory_hostname]['general']['host_packages'] }}"
     update_cache: yes
     state: latest
 
+- name: "Remove packages"
+  apt:
+    pkg: "{{ vars[inventory_hostname]['general']['host_removed_packages'] }}"
+    update_cache: yes
+    state: absent
+
+- name: "Add keyrings directory"
+  file:
+    path: /etc/apt/keyrings
+    state: directory
+    mode: "755"
+
+- name: "Add vector signed repositories"
+  ansible.builtin.get_url:
+    url: https://repositories.timber.io/public/vector/gpg.3543DB2D0A2BC4B8.key
+    dest: /etc/apt/keyrings/vector.asc
+
+- name: "Add vector repository"
+  ansible.builtin.apt_repository:
+    repo: "deb [signed-by=/etc/apt/keyrings/vector.asc] https://repositories.timber.io/public/vector/deb/debian {{ ansible_distribution_release }} main"
+    state: present
+
+- name: "Install vector"
+  ansible.builtin.apt:
+    name: vector
+    update_cache: yes
+    state: present
+
 - name: "Host is modified"
   include_tasks: host_modified.yml
   when: build_host.host_changed
@@ -71,25 +99,3 @@
     path: /var/lib/risotto/tls
     state: directory
     mode: "755"
-
-- name: "Add keyrings directory"
-  file:
-    path: /etc/apt/keyrings
-    state: directory
-    mode: "755"
-
-- name: "Add vector signed repositories"
-  ansible.builtin.get_url:
-    url: https://repositories.timber.io/public/vector/gpg.3543DB2D0A2BC4B8.key
-    dest: /etc/apt/keyrings/vector.asc
-
-- name: "Add vector repository"
-  ansible.builtin.apt_repository:
-    repo: "deb [signed-by=/etc/apt/keyrings/vector.asc] https://repositories.timber.io/public/vector/deb/debian {{ ansible_distribution_release }} main"
-    state: present
-
-- name: "Install vector"
-  ansible.builtin.apt:
-    name: vector
-    update_cache: yes
-    state: present

ansible/host_modified.yml

@@ -57,6 +57,7 @@
 
 - name: "Start services"
   when: item.value['manage'] and item.value['activate'] and item.value['doc'].endswith('.service') and not item.value['doc'].endswith('@.service') and item.value['engine']
+  ignore_errors: true
   ansible.builtin.service:
     name: "{{ item.value['doc'] }}"
     state: started

ansible/inventory.py

@@ -41,6 +41,7 @@ class RisottoInventory(object):
         parser.add_argument('--nocache', action='store_true')
         parser.add_argument('--debug', action='store_true')
         parser.add_argument('--pretty_print', action='store_true')
+        parser.add_argument('--quite', action='store_true')
         self.args = parser.parse_args()
         if self.args.debug:
             global DEBUG
@@ -75,7 +76,7 @@ class RisottoInventory(object):
                 'hosts': servers,
                 'vars': {
                     # FIXME
-#                    'ansible_ssh_host': '192.168.0.29',
+#                    'ansible_ssh_host': '192.168.0.28',
                     'ansible_ssh_user': 'root',
                     'ansible_python_interpreter': '/usr/bin/python3'
                 }
@@ -119,7 +120,7 @@ def main():
             from pprint import pprint
             from json import loads
             pprint(loads(values))
-        else:
+        elif not inv.args.quite:
             print(values)
     except Exception as err:
         if DEBUG:

ansible/sbin/make_changelog

@@ -38,12 +38,13 @@ def read_dnf_pkg_file(os_name, filename1, filename2):
                 sp_line = pkg_line.strip().split()
                 if len(sp_line) < idx_version[fidx] + 1:
                     continue
-                if sp_line[idx_pkg[fidx]] in pkgs:
-                    raise Exception(f'package already set {sp_line[0]}?')
+                pkg = sp_line[idx_pkg[fidx]]
                 version = sp_line[idx_version[fidx]]
+                #if pkg in pkgs:
+                #    raise Exception(f'package already set {pkg}?')
                 if os_name == 'debian' and version.startswith('('):
                     version = version[1:]
-                pkgs[sp_line[idx_pkg[fidx]]] = version
+                pkgs[pkg] = version
     return pkgs
 
 

ansible/sbin/update_images

@@ -23,7 +23,7 @@ ls /var/lib/risotto/images_files/ | while read image; do
     if [ -d /var/lib/risotto/images_files/"$image" ]; then
         echo
         echo "Install image $image" | tee -a /var/log/risotto/update_images.log
-        /usr/local/sbin/build_image "$image" | tee -a /var/log/risotto/update_images.log || (echo "PROBLEME" | tee -a /var/log/risotto/update_images.log; true)
+        /usr/local/sbin/build_image "$image" || echo "PROBLEME" | tee -a /var/log/risotto/update_images.log
     fi
 done
 
@@ -61,7 +61,7 @@ for nspawn in $(ls /etc/systemd/nspawn/*.nspawn); do
             sleep 1
         done
     fi
-    if [ ! -d "$MACHINE_MACHINES_DIR" ]; then
+    if [ ! -d "$MACHINE_MACHINES_DIR/etc" ]; then
         rm -f "$VERSION_MACHINE"
     fi
     diff -q "$RISOTTO_IMAGE_DIR/$IMAGE_NAME_RISOTTO_IMAGE_NAME".version "$VERSION_MACHINE" &> /dev/null || (

src/risotto/image.py

@@ -3,6 +3,7 @@ from os import listdir, makedirs
 from os.path import join, isdir, isfile, dirname
 from yaml import load as yaml_load, SafeLoader
 from tiramisu.error import PropertiesOptionError
+from .rougail import func
 #
 from .utils import RISOTTO_CONFIG
 
@@ -11,7 +12,7 @@ class ModuleCfg():
     def __init__(self, module_name):
         self.module_name = module_name
         self.dictionaries_dir = []
-        self.functions_file = []
+        self.functions_file = [func.__file__]
         self.templates_dir = []
         self.patches_dir = []
         self.extra_dictionaries = {}

src/risotto/machine.py

@@ -1,6 +1,5 @@
 from .utils import MULTI_FUNCTIONS, load_zones, value_pprint, RISOTTO_CONFIG, EXTRA_ANNOTATORS, ROUGAIL_NAMESPACE, ROUGAIL_NAMESPACE_DESCRIPTION
 from .image import Applications, Modules, valid_mandatories, applicationservice_copy
-from .rougail.annotator import calc_providers, calc_providers_global, calc_providers_dynamic, calc_providers_dynamic_follower, calc_providers_follower
 
 from rougail import RougailConfig, RougailConvert
 from os import remove, makedirs, listdir, chmod
@@ -9,7 +8,7 @@ from pickle import dump as pickle_dump, load as pickle_load
 from yaml import load as yaml_load, SafeLoader
 from ipaddress import IPv4Interface, ip_network
 #
-from tiramisu import Config, valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal, calc_value, calc_value_property_help
+from tiramisu import Config
 from rougail.utils import normalize_family
 from rougail import RougailSystemdTemplate
 from shutil import copy2, copytree, rmtree
@@ -36,20 +35,6 @@ INSTALL_CONFIG_DIR = 'configurations'
 INSTALL_TMPL_DIR= 'templates'
 INSTALL_IMAGES_DIR = 'images_files'
 INSTALL_TESTS_DIR = 'tests'
-FUNCTIONS = {'calc_providers': calc_providers,
-             'calc_providers_global': calc_providers_global,
-             'calc_providers_dynamic': calc_providers_dynamic,
-             'calc_providers_dynamic_follower': calc_providers_dynamic_follower,
-             'calc_providers_follower': calc_providers_follower,
-             'valid_network_netmask': valid_network_netmask,
-             'valid_ip_netmask': valid_ip_netmask,
-             'valid_broadcast': valid_broadcast,
-             'valid_in_network': valid_in_network,
-             'valid_not_equal': valid_not_equal,
-             'calc_value': calc_value,
-             'calc_value_property_help': calc_value_property_help,
-             'normalize_family': normalize_family,
-             }
 
 
 def copy(src_file, dst_file):
@@ -201,7 +186,6 @@ class Loader:
         cfg['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
         cfg['multi_functions'] = MULTI_FUNCTIONS
         cfg['extra_annotators'] = EXTRA_ANNOTATORS
-        cfg['internal_functions'] = list(FUNCTIONS.keys())
         cfg['force_convert_dyn_option_description'] = True
         cfg['risotto_globals'] = {}
 
@@ -267,6 +251,7 @@ class Loader:
             self.zones[zone_name] = {'domain_name': zone_domaine_name,
                                      'network': str(sub_network),
                                      'host_ip': str(network + 1),
+                                     'host_name': host_name.split('.', 1)[0],
                                      'length': length,
                                      'start_ip': str(network + 2)
                                      }
@@ -289,8 +274,10 @@ class Loader:
             # load host
             module_info = modules.get('host')
             tls_host_name = f'{server_name}.{self.zones[list(self.zones)[0]]["domain_name"]}'
+            short_host_name = host_name.split('.', 1)[0]
+            values = [f'{short_host_name}.{self.zones[zone_name]["domain_name"]}' for zone_name in zones_name]
             cfg['risotto_globals'][host_name] = {'global:server_name': host_name,
-                                                 'global:server_names': [host_name for zone in self.zones],
+                                                 'global:server_names': values,
                                                  'global:zones_name': list(self.zones),
                                                  'global:module_name': 'host',
                                                  'global:host_install_dir': abspath(INSTALL_DIR),
@@ -333,6 +320,16 @@ class Loader:
             self.modules[host_name] = modules_info
         cfg['functions_file'] = list(functions_files)
         self.tiram_obj = rougail.save(TIRAMISU_CACHE)
+        with open(TIRAMISU_CACHE, 'a') as cache:
+            cache.write(f"""#from pickle import load
+#config = Config(option_0)
+#config.property.read_only()
+#with open('{VALUES_CACHE}', 'rb') as fh:
+#    config.value.importation(load(fh))
+#with open('{INFORMATIONS_CACHE}', 'rb') as fh:
+#    config.information.importation(load(fh))
+#print(config.value.mandatory())
+""")
 
     def add_tls(self):
         zones = set()
@@ -373,19 +370,18 @@ class Loader:
 
     def tiramisu_file_to_tiramisu(self):
         # l
-        tiramisu_space = FUNCTIONS.copy()
+        tiramisu_space = {}
         try:
             exec(self.tiram_obj, None, tiramisu_space)
         except Exception as err:
-            print(self.tiram_obj)
-            raise Exception(f'unknown error when load tiramisu object {err}') from err
+            raise Exception(f'unknown error when load tiramisu object: "{err}" see the file "{TIRAMISU_CACHE}" for more details') from err
         if self.original_display_name:
             display_name = None
         else:
             display_name = tiramisu_display_name
         self.config = Config(tiramisu_space['option_0'],
-                                   display_name=display_name,
-                                   )
+                             display_name=display_name,
+                             )
 
     def load_values_and_informations(self):
         config = self.config
@@ -416,6 +412,10 @@ class Loader:
         # FIXME only one host_name is supported
         config.information.set('modules', self.modules[host_name])
         # config.information.set('modules', {module_name: module_info.depends for module_name, module_info in self.module_infos.items() if module_name in modules})
+        with open(VALUES_CACHE, 'wb') as fh:
+            pickle_dump(config.value.exportation(), fh)
+        with open(INFORMATIONS_CACHE, 'wb') as fh:
+            pickle_dump(config.information.exportation(), fh)
         config.property.add('cache')
         if self.valid_mandatories:
             messages = valid_mandatories(config)
@@ -426,10 +426,6 @@ class Loader:
                     msg += '\n'.join(variables)
                 raise Exception(msg)
         config.property.read_only()
-        with open(VALUES_CACHE, 'wb') as fh:
-            pickle_dump(config.value.exportation(), fh)
-        with open(INFORMATIONS_CACHE, 'wb') as fh:
-            pickle_dump(config.information.exportation(), fh)
 
     def set_values(self,
                    server_name,

src/risotto/rougail/annotator.py

@@ -1,136 +1,9 @@
 from rougail.annotator.variable import Walk
 from rougail.error import DictConsistencyError
-from risotto.utils import _, multi_function
+from rougail.utils import normalize_family
+from risotto.utils import _
 from warnings import warn
-
-
-def _parse_kwargs(provider, dns, kwargs, index=None):
-    if not isinstance(dns, list):
-        raise Exception('pfff')
-    values = {}
-    for key, value in kwargs.items():
-        if '_' not in key:
-            raise Exception(f'unknown attribute {key} in calc_providers_global with provider {provider}')
-        k, idx = key.rsplit('_', 1)
-        values.setdefault(idx, {})[k] = value
-    for idx, data in values.items():
-        if index is not None and int(idx) != index:
-            continue
-        if 'dns' not in data:
-            continue
-        if isinstance(data['dns'], list):
-            for ddns in data['dns']:
-                if ddns in dns:
-                    break
-            else:
-                continue
-        elif data['dns'] not in dns:
-            continue
-#        del data['dns']
-        yield data
-
-
-@multi_function
-def calc_providers_global(provider, multi, unique, value, suffix=None):
-    if suffix is not None:
-        return value[int(suffix)]
-    return value
-
-
-@multi_function
-def calc_providers_follower(provider, multi, unique, dns, leader, index, **kwargs):
-    ret = []
-    for data in _parse_kwargs(provider, dns, kwargs):
-        if 'value' not in data:
-            continue
-        if 'leader' in data:
-            if isinstance(data['leader'], list):
-                for idx, leader_iter in enumerate(data['leader']):
-                    if leader_iter == leader:
-                        ret.append(data['value'][idx])
-            elif data['leader']== leader:
-                ret.extend(data['value'])
-        else:
-            if isinstance(data['value'], list):
-                for v in data['value']:
-                    if v not in ret:
-                        ret.append(v)
-            elif data['value'] not in ret:
-                ret.append(data['value'])
-    if multi: 
-        return ret
-    if ret:
-        return ret[0]
-
-
-@multi_function
-def calc_providers_dynamic_follower(provider, multi, unique, dns, leader, index, suffix, **kwargs):
-    ret = []
-    for data in _parse_kwargs(provider, dns, kwargs):
-        if 'value' not in data:
-            continue
-        if data['dynamic'] != suffix:
-            continue
-        if 'leader' in data:
-            for idx, leader_iter in enumerate(data['leader']):
-                if leader_iter == leader:
-                    if isinstance(data['value'], list):
-                        ret.append(data['value'][idx])
-                    else:
-                        ret.append(data['value'])
-        else:
-            if isinstance(data['value'], list):
-                for v in data['value']:
-                    if v not in ret:
-                        ret.append(v)
-            elif data['value'] not in ret:
-                ret.append(data['value'])
-    if multi: 
-        return ret
-    if ret:
-        return ret[0]
-
-
-@multi_function
-def calc_providers_dynamic(provider, multi, unique, dns, suffix, **kwargs):
-    ret = []
-    for data in _parse_kwargs(provider, dns, kwargs):
-        if 'value' not in data:
-            continue
-        if data['dynamic'] != suffix:
-            continue
-        if isinstance(data['value'], list):
-            for v in data['value']:
-                if not unique or v not in ret:
-                    ret.append(v)
-        elif data['value'] not in ret:
-            ret.append(data['value'])
-    if multi: 
-        return ret
-    if ret:
-        return ret[0]
-
-
-@multi_function
-def calc_providers(provider, multi, unique, dns, **kwargs):
-    ret = []
-    for data in _parse_kwargs(provider, dns, kwargs):
-        if isinstance(data['value'], list):
-            commun_dns = list(set(data['dns']) & set(dns))
-            if len(commun_dns) == 1:
-                ret.append(data['value'][data['dns'].index(commun_dns[0])])
-                continue
-            for v in data['value']:
-                if v in ret:
-                    continue
-                ret.append(v)
-
-        elif data['value'] not in ret:
-            ret.append(data['value'])
-    if multi: 
-        return ret
-    if ret:
-        return ret[0]
+from typing import List, Tuple
 
 
 class Annotator(Walk):
@@ -138,290 +11,394 @@ class Annotator(Walk):
     def __init__(self,
                  objectspace: 'RougailObjSpace',
                  *args):
-        self.objectspace = objectspace
-        self.get_suppliers_providers()
-        self.dispatch_provider_supplier_to_zones()
-        self.dispatch_provider_to_zones()
-        self.convert_providers()
-        self.convert_suppliers()
-
-    def get_suppliers_providers(self) -> None:
-        """ get supplier informations
-        return something like:
-        {'Host': ['host1.example.net', 'host2.example.net']}
-        """
-        self.suppliers = {}
         self.providers = {}
+        self.suppliers = {}
+        self.globals = {}
+        self.provider_links = {}
+        self.providers_zone = {}
+        self.provider_maps = {}
+        self.objectspace = objectspace
+        #
+        self.get_providers_suppliers()
+        self.get_provider_links()
+        self.get_provider_maps()
+        self.convert_providers()
+        self.convert_globals()
+
+    def get_providers_suppliers(self) -> None:
+        """parse all variable and get provider and supplier informations
+        """
         for variable in self.get_variables():
-            if not hasattr(variable, 'supplier') and not hasattr(variable, 'provider'):
-                continue
-            nf_dns = variable.path.split('.', 1)[0]
-            server_name = self.objectspace.space.variables[nf_dns].doc
-            # supplier
-            if hasattr(variable, 'supplier') and ':' not in variable.supplier:
-                server_names = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names']
-                zones = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name']
-                s_data = {'option': variable,
-                          'dns': server_name,
-                          'path_prefix': nf_dns,
-                          'server_names': server_names,
-                          'zones': zones,
-                          'providers_zone': {},
-                          }
-                if variable.supplier != 'Host' and not variable.supplier.startswith('Host:') and not variable.supplier.startswith('global:'):
-                    if 'global:provider_zone' in self.objectspace.rougailconfig['risotto_globals'][server_name]:
-                        s_data['provider_zone'] = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:provider_zone']
-                self.suppliers.setdefault(variable.supplier, []).append(s_data)
-                if not hasattr(variable, 'information'):
-                    variable.information = self.objectspace.information(variable.xmlfiles)
-                variable.information.supplier = variable.supplier
-            # provider
-            if hasattr(variable, 'provider'):
-                provider_name = variable.provider
-                p_data = {'option': variable,
-                          'dns': server_name,
-                          'provider_name': provider_name,
-                          'path_prefix': nf_dns,
-                          'suppliers_zone': {},
-                          }
-                if variable.provider != 'Host' and not variable.provider.startswith('Host:') and not variable.provider.startswith('global:'):
-                    if 'global:provider_zone' in self.objectspace.rougailconfig['risotto_globals'][server_name]:
-                        p_data['provider_zone'] = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:provider_zone']
-                if self.objectspace.rougailconfig['risotto_globals'][server_name]['global:module_name'] == 'host':
-                    server_names = [server_name]
-                else:
-                    server_names = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names']
-                p_data['server_names'] = server_names
-                if self.objectspace.rougailconfig['risotto_globals'][server_name]['global:module_name'] != 'host':
-                    p_data['zones'] = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name']
-                if ':' in provider_name:
-                    p_data['is_main_provider'] = False
-                    provider = provider_name.rsplit(':', 1)[0]
-                else:
-                    p_data['is_main_provider'] = True
-                    provider = variable.provider
-                self.providers.setdefault(provider, []).append(p_data)
+            for type_ in 'provider', 'supplier':
+                if hasattr(variable, type_):
+                    provider_name = getattr(variable, type_)
+                    # add information with provider/supplier name
+                    if not hasattr(variable, 'information'):
+                        variable.information = self.objectspace.information(variable.xmlfiles)
+                    setattr(variable.information, type_, provider_name)
 
-    def dispatch_provider_supplier_to_zones(self):
-        """calculate zone where provider and supplier communicate
-        """
-        self.providers_zone = {}
-        for provider_name, p_datas in self.providers.items():
-            for p_data in p_datas:
-                if provider_name in ['global', 'Host'] or provider_name not in self.suppliers:
-                    continue
-                if not 'provider_zone' in p_data:
-                    provider_zone = None
-                else:
-                    provider_zone = p_data['provider_zone']
-                for s_data in self.suppliers[provider_name]:
-                    if not provider_zone:
-                        if provider_name.endswith('Client'):
-                            p_server = provider_name[0:-6]
-                            if p_server not in self.providers:
-                                continue
-                            for p_data_t in self.providers[p_server]:
-                                if p_data_t['dns'] == s_data['dns'] and 'provider_zone' in p_data_t:
-                                    zone = p_data_t['provider_zone']
-                                    break
-                            else:
-                                continue
-                        else:
-                            continue
+                    # construct self.globals, self.suppliers and self.providers dictionnaries
+                    provider_prefix, provider_suffix = self._cut_out_provider_name(provider_name)
+                    dns = self.objectspace.space.variables[variable.path_prefix].doc
+                    if provider_prefix == 'global':
+                        if type_ == 'supplier':
+                            raise DictConsistencyError(f'{type_} {provider_name} in {dns} not allowed', 0, variable.xmlfiles)
+                        obj = self.globals
+                    elif type_ == 'supplier':
+                        obj = self.suppliers
                     else:
-                        zone = provider_zone
-                    if zone not in s_data['zones']:
-                        continue
-                    s_data['providers_zone'][provider_name] = zone
-                    p_data['suppliers_zone'].setdefault(provider_name, {})[s_data['dns']] = zone
-                    self.providers_zone.setdefault(zone, set()).add(provider_name)
+                        obj = self.providers
+                    sub_obj = obj.setdefault(provider_prefix, {}).setdefault(dns, {})
+                    if provider_suffix in sub_obj:
+                        raise DictConsistencyError(f'multiple {type_} {provider_name} in {dns}', 0, sub_obj[provider_suffix].xmlfiles + variable.xmlfiles)
+                    sub_obj[provider_suffix] = variable
 
-    def dispatch_provider_to_zones(self):
-        """ add information with provider zone domain name
+    def _cut_out_provider_name(self,
+                               provider_name: str,
+                               ) -> Tuple[str, str]:
+        """get provider_name and return provider_prefix and provider_suffix
         """
-        self.providers_zone = {}
-        for provider_name, p_datas in self.providers.items():
-            for p_data in p_datas:
-                if provider_name in ['global', 'Host'] or provider_name not in self.suppliers:
+        if ':' in provider_name:
+            provider_prefix, provider_suffix = provider_name.split(':', 1)
+        else:
+            provider_prefix = provider_name
+            provider_suffix = None
+        return provider_prefix, provider_suffix
+
+    def get_provider_links(self):
+        """Search link between providers
+           'ProviderPrefix': {'provider_dns': ['supplier_dns_1',
+                                               'supplier_dns_2',
+                                               'supplier_dns_3']}
+        """
+        for provider_prefix, providers_dns in self.providers.items():
+            self.provider_links[provider_prefix] = {}
+            for provider_dns, providers_suffix in providers_dns.items():
+                if None not in providers_suffix:
+                    # it's a reverse provider!
                     continue
-                if not 'provider_zone' in p_data:
+                if provider_prefix != 'Host':
+                    provider_zone = self.objectspace.rougailconfig['risotto_globals'][provider_dns]['global:provider_zone']
+                if provider_prefix not in self.suppliers:
                     continue
-                provider_zone = p_data['provider_zone']
-                family = self.objectspace.paths.get_variable(f"providers",
-                                                             namespace=self.objectspace.rougailconfig['variable_namespace'],
-                                                             force_path_prefix=p_data['path_prefix'],
+                for supplier_dns, suppliers_suffix in self.suppliers[provider_prefix].items():
+                    if provider_dns == supplier_dns:
+                        continue
+                    if provider_prefix == 'Host':
+                        provider_zone = self.objectspace.rougailconfig['risotto_globals'][supplier_dns]['global:zones_name'][0]
+                    if provider_zone not in self.objectspace.rougailconfig['risotto_globals'][supplier_dns]['global:zones_name']:
+                        continue
+                    self.provider_links[provider_prefix].setdefault(provider_dns, []).append(supplier_dns)
+
+    def get_provider_maps(self):
+        """relation between provider_prefix and provider_suffix
+           'provider_prefix': {'normal': {None, 'provider_suffix_1', 'provider_suffix_2'},
+                               'reverse': {'provider_suffix_3', 'provider_suffix_4'}}
+        """
+        for provider_prefix, providers_dns in self.provider_links.items():
+            self.provider_maps[provider_prefix] = {'normal': set(), 'reverse': set()}
+            for provider_dns, suppliers_dns in providers_dns.items():
+                for supplier_dns in suppliers_dns:
+                    if supplier_dns in self.providers[provider_prefix] and None in self.providers[provider_prefix][supplier_dns]:
+                        #exists?
+                        continue
+                    # get prefixes
+                    prefixes = set(self.providers[provider_prefix][provider_dns]) & set(self.suppliers[provider_prefix][supplier_dns])
+                    self.provider_maps[provider_prefix]['normal'] |= prefixes
+                    # get suffixes
+                    if supplier_dns not in self.providers[provider_prefix]:
+                        continue
+                    suffixes = set(self.providers[provider_prefix][supplier_dns]) & set(self.suppliers[provider_prefix][provider_dns])
+                    self.provider_maps[provider_prefix]['reverse'] |= suffixes
+
+    def convert_providers(self) -> None:
+        """Convert providers informations to default values or fills
+        """
+        for provider_prefix, providers_dns in self.provider_links.items():
+            for provider_dns, suppliers_dns in providers_dns.items():
+                for provider_suffix in self.provider_maps[provider_prefix]['normal']:
+                    self._convert_providers_normal(provider_prefix,
+                                                   provider_suffix,
+                                                   provider_dns,
+                                                   suppliers_dns,
+                                                   )
+                for provider_suffix in self.provider_maps[provider_prefix]['reverse']:
+                    self._convert_providers_reverse(provider_prefix,
+                                                    provider_suffix,
+                                                    provider_dns,
+                                                    suppliers_dns,
+                                                    )
+
+    def _convert_providers_normal(self,
+                                  provider_prefix: str,
+                                  provider_suffix: str,
+                                  provider_dns: str,
+                                  suppliers_dns: dict,
+                                  ) -> None:
+        if provider_prefix != 'Host':
+            provider_zone = self.objectspace.rougailconfig['risotto_globals'][provider_dns]['global:provider_zone']
+            provider_option_dns = self._get_dns_from_provider_zone(provider_dns, provider_zone)
+        variable = self.providers[provider_prefix][provider_dns][provider_suffix]
+        if hasattr(variable, 'value'):
+            raise DictConsistencyError(f'variable {variable.path} has a provider and a value', 0, variable.xmlfiles)
+        suppliers_var = {}
+        for supplier_dns in suppliers_dns:
+            if provider_suffix not in self.suppliers[provider_prefix][supplier_dns]:
+                continue
+            if provider_prefix == 'Host':
+                provider_zone = self.objectspace.rougailconfig['risotto_globals'][supplier_dns]['global:zones_name'][0]
+                provider_option_dns = self._get_dns_from_provider_zone(provider_dns, provider_zone)
+            supplier_variable = self.suppliers[provider_prefix][supplier_dns][provider_suffix]
+            supplier_option_dns = self._get_dns_from_provider_zone(supplier_dns, provider_zone)
+            suppliers_var[supplier_option_dns] = supplier_variable
+        if provider_suffix:
+            AddFill(self.objectspace,
+                    provider_prefix,
+                    provider_suffix,
+                    provider_option_dns,
+                    variable,
+                    suppliers_var,
+                    False,
+                    supplier_variable.path_prefix,
+                    )
+        else:
+            self._set_provider_supplier(provider_option_dns,
+                                        variable,
+                                        suppliers_var,
+                                        )
+
+    def _convert_providers_reverse(self,
+                                   provider_prefix: str,
+                                   provider_suffix: str,
+                                   provider_dns: str,
+                                   suppliers_dns: dict,
+                                   ) -> None:
+        if provider_prefix != 'Host':
+            provider_zone = self.objectspace.rougailconfig['risotto_globals'][provider_dns]['global:provider_zone']
+            provider_option_dns = self._get_dns_from_provider_zone(provider_dns, provider_zone)
+        variable = self.suppliers[provider_prefix][provider_dns][provider_suffix]
+        if hasattr(variable, 'value'):
+            raise DictConsistencyError(f'variable {variable.path} has a provider and a value', 0, variable.xmlfiles)
+        for supplier_dns in suppliers_dns:
+            if provider_prefix == 'Host':
+                provider_zone = self.objectspace.rougailconfig['risotto_globals'][supplier_dns]['global:zones_name'][0]
+                provider_option_dns = self._get_dns_from_provider_zone(provider_dns, provider_zone)
+            supplier_variable = self.providers[provider_prefix][supplier_dns][provider_suffix]
+            supplier_option_dns = self._get_dns_from_provider_zone(supplier_dns, provider_zone)
+            AddFill(self.objectspace,
+                    provider_prefix,
+                    provider_suffix,
+                    supplier_option_dns,
+                    supplier_variable,
+                    {provider_option_dns: variable},
+                    True,
+                    supplier_variable.path_prefix,
+                    )
+
+    def _get_dns_from_provider_zone(self,
+                                    dns,
+                                    zone,
+                                    ) -> str:
+        risotto_global = self.objectspace.rougailconfig['risotto_globals'][dns]
+        index = risotto_global['global:zones_name'].index(zone)
+        return risotto_global['global:server_names'][index]
+
+    def _set_provider_supplier(self,
+                               provider_dns: str,
+                               variable,
+                               suppliers,
+                               ) -> None:
+        # suffix is None so not a client and there only one provider
+        # the value of this variable is the list of suppliers DNS name
+        if not variable.multi:
+            raise DictConsistencyError(f'"{variable.name}" is a provider and must be a multi', 0, variable.xmlfiles)
+        variable.default = list(suppliers)
+        # suffix is None so the supplier values are provider DNS
+        for sub_variable in suppliers.values():
+            #FIXME
+            #if hasattr(option, 'value'):
+            #    raise DictConsistencyError(f'"{option.name}" is a supplier and cannot have value', 0, option.xmlfiles)
+            if sub_variable.multi:
+                raise DictConsistencyError(f'"{sub_variable.name}" is a supplier and mustnot be a multi', 0, sub_variable.xmlfiles)
+            sub_variable.default = provider_dns
+
+    def convert_globals(self):
+        """Convert providers global informations to default values or fills
+        """
+        provider_prefix = 'global'
+        for provider_dns, providers_suffix in self.globals[provider_prefix].items():
+            for provider_suffix, variable in providers_suffix.items():
+                provider_name = f'{provider_prefix}:{provider_suffix}'
+                if provider_name not in self.objectspace.rougailconfig['risotto_globals'][provider_dns]:
+                    raise DictConsistencyError(f'cannot find {provider_name} for variable {variable.path}, should be in {list(self.objectspace.rougailconfig["risotto_globals"][provider_dns])}', 0, variable.xmlfiles)
+                provider_values = self.objectspace.rougailconfig['risotto_globals'][provider_dns][provider_name]
+                if isinstance(provider_values, list) and self.objectspace.paths.is_dynamic(variable):
+                    if variable.multi:
+                        raise DictConsistencyError(f'variable {variable.path} has provider {provider_name} and is in dynamic family so must not be a multi', 0, variable.xmlfiles)
+                    self._set_global_dynamic_option(variable, provider_values)
+                else:
+                    if isinstance(provider_values, list) and not variable.multi:
+                        raise DictConsistencyError(f'variable {variable.path} has provider {provider_name} which is a multi', 0, variable.xmlfiles)
+                    if not isinstance(provider_values, list):
+                        if variable.multi:
+                            raise DictConsistencyError(f'variable {variable.path} has provider {provider_name} which is not a multi', 0, variable.xmlfiles)
+                        provider_values = [provider_values]
+
+                    variable.value = []
+                    for provider_value in provider_values:
+                        value = self.objectspace.value(variable.xmlfiles)
+                        value.name = provider_value
+                        if isinstance(provider_value, bool):
+                            value.type = 'boolean'
+                        variable.value.append(value)
+
+    def _set_global_dynamic_option(self,
+                                   variable: 'self.objectspace.Variable',
+                                   values: List[str],
+                                   ):
+        fill = self.objectspace.fill(variable.xmlfiles)
+        new_target = self.objectspace.target(variable.xmlfiles)
+        new_target.name = variable.name
+        fill.target = [new_target]
+        fill.namespace = variable.namespace
+        fill.index = 0
+        fill.name = 'risotto_providers_global'
+        param1 = self.objectspace.param(variable.xmlfiles)
+        param1.text = values
+        param2 = self.objectspace.param(variable.xmlfiles)
+        param2.type = 'suffix'
+        fill.param = [param1, param2]
+        if not hasattr(self.objectspace.space.variables[variable.path_prefix].constraints, 'fill'):
+            self.objectspace.space.variables[variable.path_prefix].constraints.fill = []
+        self.objectspace.space.variables[variable.path_prefix].constraints.fill.append(fill)
+
+
+class AddFill:
+    """Add fill for variable
+    """
+    def __init__(self,
+                 objectspace,
+                 provider_prefix,
+                 provider_suffix,
+                 provider_dns,
+                 variable,
+                 suppliers,
+                 reverse,
+                 path_prefix,
+                 ) -> None:
+        self.objectspace = objectspace
+        self.provider_dns = provider_dns
+        self.variable = variable
+        self.path_prefix = path_prefix
+        self.suppliers = suppliers
+        self.create_fill()
+        if reverse:
+            self.param_reverse()
+        else:
+            if self.objectspace.paths.is_dynamic(self.variable):
+                self.param_dynamic()
+            elif self.variable.multi:
+                self.param_multi()
+            else:
+                provider_name = f'{provider_prefix}:{provider_suffix}'
+                raise DictConsistencyError(f'provider "{provider_name}" options must be in dynamic option or must be a multiple', 0, self.variable.xmlfiles)
+            if self.objectspace.paths.is_follower(self.variable):
+                self.param_follower()
+        self.end()
+
+    def create_fill(self) -> None:
+        self.fill = self.objectspace.fill(self.variable.xmlfiles)
+        new_target = self.objectspace.target(self.variable.xmlfiles)
+        new_target.name = self.variable
+        self.fill.target = [new_target]
+        self.fill.namespace = self.variable.namespace
+        self.fill.index = 0
+        self.fill.param = []
+
+    def param_reverse(self) -> None:
+        self.fill.name = 'risotto_flatten_values_client'
+        #
+        if self.objectspace.paths.is_follower(self.variable):
+            multi = self.variable.multi is True
+        else:
+            multi = self.variable.multi is not False
+        param = self.objectspace.param(self.variable.xmlfiles)
+        param.text = multi
+        param.type = 'boolean'
+        self.fill.param.append(param)
+        for dns, variable in self.suppliers.items():
+            param = self.objectspace.param(variable.xmlfiles)
+            param.text = variable
+            param.propertyerror = False
+            param.type = 'variable'
+            param.suffix = normalize_family(self.provider_dns)
+            namespace = variable.namespace
+            family_path = self.objectspace.paths.get_variable_family_path(param.text.path, 
+                                                                          namespace,
+                                                                          force_path_prefix=self.variable.path_prefix,
+                                                                          )
+            param.family = self.objectspace.paths.get_family(family_path,
+                                                             namespace,
+                                                             self.variable.path_prefix,
                                                              )
-                if not hasattr(family, 'information'):
-                    family.information = self.objectspace.information(family.xmlfiles)
-                name_in_zone = p_data['server_names'][p_data['zones'].index(provider_zone)]
-                setattr(family.information, provider_name, name_in_zone)
-                setattr(family.information, f'{provider_name}:zone', provider_zone)
+            self.fill.param.append(param)
 
-    def convert_providers(self):
-        for provider_name, providers_data in self.providers.items():
-            for provider_data in providers_data:
-                if provider_name != 'global' and provider_name not in self.suppliers:
-                    continue
-                # create a fill for this variable
-                variable = provider_data['option']
-                fill = self.objectspace.fill(variable.xmlfiles)
-                new_target = self.objectspace.target(variable.xmlfiles)
-                new_target.name = variable
-                fill.target = [new_target]
-                if provider_name == 'global':
-                    fill.name = 'calc_providers_global'
-                elif self.objectspace.paths.is_dynamic(variable):
-                    if self.objectspace.paths.is_follower(variable):
-                        fill.name = 'calc_providers_dynamic_follower'
-                    else:
-                        fill.name = 'calc_providers_dynamic'
-                elif self.objectspace.paths.is_follower(variable):
-                    fill.name = 'calc_providers_follower'
-                else:
-                    fill.name = 'calc_providers'
-                fill.namespace = variable.namespace
-                fill.index = 0
-                # first parameter: the provider name (something link Host:incoming_ports)
-                param = self.objectspace.param(variable.xmlfiles)
-                param.name = 'provider'
-                param.text = provider_data['provider_name']
-                fill.param = [param]
-                # second parameter: current variable is a multi variable?
-                param = self.objectspace.param(variable.xmlfiles)
-                param.name = 'multi'
-                param.text = variable.multi
-                param.type = 'boolean'
-                fill.param.append(param)
-                #
-                param = self.objectspace.param(variable.xmlfiles)
-                param.name = 'unique'
-                param.text = variable.unique != "False"
-                param.type = 'boolean'
-                fill.param.append(param)
-                #
-                if self.objectspace.paths.is_follower(variable):
-                    param = self.objectspace.param(variable.xmlfiles)
-                    param.name = 'leader'
-                    param.text = self.objectspace.paths.get_leader(variable)
-                    param.propertyerror = False
-                    param.type = 'variable'
-                    fill.param.append(param)
-                    try:
-                        leader_provider = self.objectspace.paths.get_leader(variable).provider
-                    except:
-                        leader_provider = None
-                    #
-                    param = self.objectspace.param(variable.xmlfiles)
-                    param.name = 'index'
-                    param.type = 'index'
-                    fill.param.append(param)
-                if self.objectspace.paths.is_dynamic(variable):
-                    # if dynamic: current suffix
-                    # and add current DNS name, this is useful to known if supplier is link to this provider
-                    param = self.objectspace.param(variable.xmlfiles)
-                    param.name = 'suffix'
-                    param.type = 'suffix'
-                    fill.param.append(param)
-                if provider_name != 'global':
-                    param = self.objectspace.param(variable.xmlfiles)
-                    param.name = 'dns'
-                    param.text = provider_data['server_names']
-                    fill.param.append(param)
-                if provider_name == 'global':
-                    param = self.objectspace.param(variable.xmlfiles)
-                    param.name = 'value'
-                    if provider_data['provider_name'] in self.objectspace.rougailconfig['risotto_globals'][provider_data['dns']]:
-                        value = self.objectspace.rougailconfig['risotto_globals'][provider_data['dns']][provider_data['provider_name']]
-                        param.text = value
-                        if isinstance(value, bool):
-                            param.type = 'boolean'
-                    else:
-                        param.text = provider_data['provider_name']
-                        param.type = 'information'
-                    fill.param.append(param)
-                else:
-                    # parse all supplier link to current provider
-                    for idx, data in enumerate(self.suppliers[provider_name]):
-                        if 'zones' in provider_data:
-                            if provider_name not in data['providers_zone']:
-                                continue
-                            zone = data['providers_zone'][provider_name]
-                            zidx = data['zones'].index(zone)
-                            dns = data['server_names'][zidx]
-                        else:
-                            dns = data['dns']
-                        option = data['option']
-                        # if not provider, get the true option that we want have value
-                        if not provider_data['is_main_provider']:
-                            path_prefix = data['path_prefix']
-                            try:
-                                supplier_option = self.objectspace.paths.get_supplier(f'supplier:{provider_data["provider_name"]}', path_prefix)
-                            except KeyError:
-                                #warn(f'cannot find supplier "{provider_name}" for "{dns}"')
-                                continue
-                        # first of all, get the supplier name
-                        param = self.objectspace.param(variable.xmlfiles)
-                        param.name = f'dns_{idx}'
-                        param.text = option
-                        param.propertyerror = False
-                        param.type = 'variable'
-                        fill.param.append(param)
-                        if not provider_data['is_main_provider'] and \
-                                self.objectspace.paths.is_follower(variable):
-                            param = self.objectspace.param(variable.xmlfiles)
-                            param.name = f'leader_{idx}'
-                            if fill.name == 'calc_providers_follower':
-                                param.text = dns
-                            else:
-                                if self.objectspace.paths.is_follower(supplier_option):
-                                    param.text = self.objectspace.paths.get_leader(supplier_option)
-                                else:
-                                    param.text = self.objectspace.paths.get_supplier(f'supplier:{leader_provider}', path_prefix)
-                                param.propertyerror = False
-                                param.type = 'variable'
-                            fill.param.append(param)
-                        # get the current DNS name for dynamic variable
-                        if self.objectspace.paths.is_dynamic(variable):
-                            param = self.objectspace.param(variable.xmlfiles)
-                            param.name = f'dynamic_{idx}'
-                            param.text = dns
-                            fill.param.append(param)
-                        # get the current value!
-                        param = self.objectspace.param(variable.xmlfiles)
-                        param.name = f'value_{idx}'
-                        if provider_data['is_main_provider']:
-                            param.text = dns
-                        else:
-                            param.text = supplier_option
-                            param.propertyerror = False
-                            param.type = 'variable'
-                        fill.param.append(param)
-                if not hasattr(self.objectspace.space.variables[provider_data['path_prefix']], 'constraints'):
-                    self.objectspace.space.variables[provider_data['path_prefix']].constraints = self.objectspace.constraints(None)
-                if not hasattr(self.objectspace.space.variables[provider_data['path_prefix']].constraints, 'fill'):
-                    self.objectspace.space.variables[provider_data['path_prefix']].constraints.fill = []
-                self.objectspace.space.variables[provider_data['path_prefix']].constraints.fill.append(fill)
+    def param_dynamic(self) -> None:
+        self.fill.name = 'risotto_dyn_values'
+        #
+        param = self.objectspace.param(self.variable.xmlfiles)
+        param.type = 'suffix'
+        self.fill.param.append(param)
+        #
+        param = self.objectspace.param(self.variable.xmlfiles)
+        param.text = self.variable.unique != "False"
+        param.type = 'boolean'
+        self.fill.param.append(param)
+        #
+        if self.objectspace.paths.is_follower(self.variable):
+            multi = self.variable.multi is True
+        else:
+            multi = self.variable.multi is not False
+        param = self.objectspace.param(self.variable.xmlfiles)
+        param.text = multi
+        param.type = 'boolean'
+        self.fill.param.append(param)
+        for dns, variable in self.suppliers.items():
+            #
+            param = self.objectspace.param(variable.xmlfiles)
+            param.text = variable
+            param.name = normalize_family(dns)
+            param.propertyerror = False
+            param.type = 'variable'
+            self.fill.param.append(param)
 
-    def convert_suppliers(self):
-        for supplier_name, s_datas in self.suppliers.items():
-            if supplier_name == 'Host':
-                continue
-            for s_data in s_datas:
-                if supplier_name not in self.providers:
-                    continue
-                for p_data in self.providers[supplier_name]:
-                    if s_data['dns'] == p_data['dns']:
-                        # supplier and provider are in same machine
-                        continue
-                    if supplier_name not in p_data['suppliers_zone'] or s_data['dns'] not in p_data['suppliers_zone'][supplier_name]:
-                        continue
-                    # get the DNS name in supplier zone
-                    zone = p_data['suppliers_zone'][supplier_name][s_data['dns']]
-                    if zone not in p_data['zones']:
-                        continue
-                    zidx = p_data['zones'].index(zone)
-                    dns = p_data['server_names'][zidx]
-                    new_value = self.objectspace.value(None)
-                    new_value.name = dns
-                    s_data['option'].value = [new_value]
-                    break
+    def param_multi(self) -> None:
+        self.fill.name = 'risotto_flatten_values'
+        #
+        if self.objectspace.paths.is_follower(self.variable):
+            multi = self.variable.multi is True
+        else:
+            multi = self.variable.multi is not False
+        param = self.objectspace.param(self.variable.xmlfiles)
+        param.text = multi
+        param.type = 'boolean'
+        self.fill.param.append(param)
+        for dns, variable in self.suppliers.items():
+            param = self.objectspace.param(variable.xmlfiles)
+            param.text = variable
+            param.propertyerror = False
+            param.type = 'variable'
+            self.fill.param.append(param)
+
+    def param_follower(self):
+        param = self.objectspace.param(self.variable.xmlfiles)
+        param.name = 'follower_index'
+        param.type = 'index'
+        self.fill.param.append(param)
+
+    def end(self):
+        if not hasattr(self.objectspace.space.variables[self.path_prefix], 'constraints'):
+            self.objectspace.space.variables[self.path_prefix].constraints = self.objectspace.constraints(None)
+        if not hasattr(self.objectspace.space.variables[self.path_prefix].constraints, 'fill'):
+            self.objectspace.space.variables[self.path_prefix].constraints.fill = []
+        self.objectspace.space.variables[self.path_prefix].constraints.fill.append(self.fill)

src/risotto/rougail/func.py

@@ -0,0 +1,61 @@
+from risotto.utils import multi_function as _multi_function
+from rougail.utils import normalize_family
+from tiramisu import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal, calc_value, calc_value_property_help
+
+
+@_multi_function
+def risotto_providers_global(value, suffix=None):
+    if suffix is not None:
+        return value[int(suffix)]
+    return value
+
+
+@_multi_function
+def risotto_flatten_values(multi, *args, follower_index=None):
+    values = []
+    #if follower_index is None:
+    #    for arg in args:
+    #        if isinstance(arg, list):
+    #            values.extend(arg)
+    #        else:
+    #            values.append(arg)
+    #else:
+    values = args
+    if follower_index is not None and len(values) > follower_index:
+        values = values[follower_index]
+    elif not multi:
+        if not values:
+            values = None
+        if len(values) == 1:
+            values = values[0]
+    return values
+
+
+@_multi_function
+def risotto_flatten_values_client(multi, *args):
+    values = []
+    for arg in args:
+        if isinstance(arg, list):
+            values.extend(arg)
+        else:
+            values.append(arg)
+    if not multi:
+        if not values:
+            values = None
+        if len(values) == 1:
+            values = values[0]
+    return values
+
+
+@_multi_function
+def risotto_dyn_values(suffix, unique, multi, follower_index=None, **kwargs):
+    values = kwargs.get(normalize_family(suffix), [] if multi else None)
+    if not multi and follower_index is not None and isinstance(values, list) and len(values) > follower_index:
+        values = values[follower_index]
+    if isinstance(values, list) and unique:
+        values_ = []
+        for val in values:
+            if val not in values_:
+                values_.append(val)
+        values = values_
+    return values