risotto/sbin/risotto_auto_doc

#!/usr/bin/env python3


from os import listdir, unlink
from os.path import isdir, isfile, join, abspath
from tabulate import tabulate
from sys import argv
from yaml import dump
from ruamel.yaml import YAML
from glob import glob

from rougail import RougailConfig
from rougail.convert import RougailConvert
from rougail.objspace import RootRougailObject
from rougail.annotator.family import Mode
from risotto.utils import EXTRA_ANNOTATORS, ROUGAIL_NAMESPACE, ROUGAIL_NAMESPACE_DESCRIPTION
from risotto.image import load_application_service
from risotto.rougail import func


rougailconfig = RougailConfig
rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE
rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION


ROUGAIL_VARIABLE_TYPE = 'https://forge.cloud.silique.fr/risotto/rougail/src/branch/main/doc/variable/README.md#le-type-de-la-variable'


def add_title_family(elts, dico, modes):
    is_dynamic = False
    for idx, elt in enumerate(elts):
        description = elt.doc
        if not idx:
            description = description.capitalize()
        space = idx + 3
        title = '#' * space + f' {description}'
        if title not in dico:
            if not is_dynamic and hasattr(elt, 'suffixes') and elt.suffixes:
                is_dynamic = True
            if is_dynamic:
                title += '*suffix value*'
            dico[title] = {'help': '', 'type': 'optiondescription', 'level': idx + 1}
            if hasattr(elt, 'information') and hasattr(elt.information, 'help'):
                help_ = elt.information.help
                if not help_.endswith('.'):
                    help_ += '.'
                dico[title]['help'] += help_
            if hasattr(elt, 'suffixes') and elt.suffixes:
                dico[title]['suffixes'] = elt.suffixes.path
                if dico[title]['help']:
                    dico[title]['help'] += ' '
                dico[title]['help'] += f'This a dynamic family generated from the variable "{elt.suffixes.path}".'
            if hasattr(elt, 'leadership') and elt.leadership:
                if dico[title]['help']:
                    dico[title]['help'] += ' '
                dico[title]['help'] += f'This family is a leadership.'
            add_mode(elt, dico[title], modes)


def add_parameter_variable(idx, child, is_leadership, type_, name):
    if type_ == 'variable':
        var_path = child.xmlfiles[-1].split('/', 2)[-1]
        parameter = f'**[{name}]({var_path})**'
    elif type_ == 'provider':
        parameter = f'**{name}**'
    subparameter = ''
    if hasattr(child, 'properties') and 'mandatory' in child.properties:
        subparameter = 'mandatory'
    if ((idx == 0 or not is_leadership) and child.multi is True) or (idx != 0 and is_leadership and child.multi == 'submulti'):
        if subparameter:
            subparameter += ', '
        subparameter += 'multiple'
    if subparameter:
        parameter += f'<br/>{subparameter}'
    var_type = f'**Type:** [`{child.type}`]({ROUGAIL_VARIABLE_TYPE})'
    return f'{parameter}<br/>{var_type}'


def add_title_variable(child, is_leadership, is_dynamic, objectspace):
    description = f'{child.doc}'
    if is_dynamic:
       description += '*suffix value*'
    if not description.endswith('.'):
        description += '.'
    if hasattr(child, 'information') and hasattr(child.information, 'help'):
        if description:
            description += '<br/>'
        help_ = child.information.help
        if not help_.endswith('.'):
            help_ += '.'
        description += help_
    if hasattr(child, 'default'):
        default = child.default
        if isinstance(default, objectspace.value):
            default = '*calculated*'
        if isinstance(default, list):
            default = '<ul>' + ''.join([f'<li>{val}</li>' for val in default]) + '</ul>'
    elif is_leadership and hasattr(child, 'default_multi'):
        default = child.default_multi
    else:
        default = None
    if hasattr(child, 'choice'):
        if description:
            description += '<br/>'
        description += '**Choices:**'
        for choice in child.choice:
            description += f'<br/>- `{choice.name}`'
            if choice.name == default:
                description += ' ← default'
    elif default is not None:
        if description:
            description += '<br/>'
        description += f'**Default:** {default}'
    if hasattr(child, 'test'):
        if description:
            description += '<br/>'
        description += f'**Example:** {child.test}'
    return description


def add_mode(child, values, modes, parent=None, is_leadership=False):
    for mode in modes:
        if mode in child.properties:
            break
    else:
        raise Exception(f'cannot find mode for {child.path}')
    if is_leadership:
        for leadership_mode in modes:
            if leadership_mode in parent.properties:
                break
        else:
            raise Exception(f'cannot find mode for {parent.path}')
        if modes[mode] < modes[leadership_mode]:
            mode = leadership_mode
    values['mode'] = mode


def parse(applicationservice,
          elts,
          dico,
          providers_suppliers,
          hidden,
          dynamic,
          objectspace,
          mandatory_values_without_values,
          modes,
          is_dynamic,
          dynamic_path,
          is_service,
          suppliers_variables,
          ):
    elt = elts[-1]
    if not hidden:
        hidden = hasattr(elt, 'properties') and ('hidden' in elt.properties or 'disabled' in elt.properties)
    if not dynamic:
        dynamic = hasattr(elt, 'suffixes')
    is_leadership = hasattr(elt, 'leadership') and elt.leadership is True
    if not is_dynamic:
        is_dynamic = hasattr(elt, 'suffixes') and elt.suffixes is not None
    dynamic_path += elt.name
    if is_dynamic:
        dynamic_path += "*suffix value*"
    dynamic_path += '.'
    leader_is_mandatory = False
    first_variable = True
    for children in vars(elt).values():
        if isinstance(children, dict):
            children = list(children.values())
        if not isinstance(children, list):
            continue
        for idx, child in enumerate(children):
            if isinstance(child, objectspace.property_) or \
                    not isinstance(child, RootRougailObject):
                continue
            if isinstance(child, objectspace.variable):
                if hasattr(child, 'provider') and child.provider and not child.provider.startswith('global:') and not child.provider.startswith('Host:'):
                    provider_prefix = child.provider
                    if ':' in provider_prefix:
                        provider_prefix = provider_prefix.split(':', 1)[0]
                    if provider_prefix != 'Host':
                        if child.doc == child.name:
                            description = None
                        else:
                            parameter = add_parameter_variable(idx, child, is_leadership, 'provider', child.provider)
                            description = add_title_variable(child, is_leadership, is_dynamic, objectspace)
                        suppliers_variables.setdefault(provider_prefix, {})[child.provider] = {'parameter': parameter, 'description': description}
                        if provider_prefix == child.provider:
                            doc = child.doc
                            if not doc.endswith('.'):
                                doc += '.'
                            suppliers_variables[provider_prefix]['doc'] = doc
                if not hidden and (not hasattr(child, 'properties') or ('hidden' not in child.properties and not 'disabled' in child.properties)):
                    if is_service:
                        if hasattr(child, 'provider'):
                            provider = child.provider
                            if ':' not in provider:
                                providers_suppliers['providers'].setdefault(provider, []).append(applicationservice)
                        if hasattr(child, 'supplier') and child.supplier:
                            supplier = child.supplier
                            if ':' not in supplier:
                                providers_suppliers['suppliers'].setdefault(supplier, []).append(applicationservice)
                                if 'mandatory' in child.properties:
                                    providers_suppliers['mandatory_suppliers'].setdefault(supplier, []).append(applicationservice)
                    if hasattr(child, 'supplier') and child.supplier and ':' not in child.supplier:
                        raise Exception(f'Main supplier variable "{child.path}" in {child.xmlfiles} must be hidden')
                    if hasattr(child, 'provider') and child.provider and not child.provider.startswith('Host:'):
                        raise Exception(f'Provider variable "{child.path}" in {child.xmlfiles} must be hidden')
                    if first_variable:
                        add_title_family(elts, dico, modes)
                        first_variable = False
                    sub_dynamic_path = dynamic_path + child.name
                    if is_dynamic:
                        sub_dynamic_path += "*suffix value*"
                    if not idx and is_leadership and hasattr(child, 'properties') and 'mandatory' in child.properties:
                        leader_is_mandatory = True
                    if hasattr(child, 'properties') and 'mandatory' in child.properties and \
                            mandatory_values_without_values is not None and \
                            not hidden and \
                            not dynamic and \
                            not hasattr(child, 'default') and \
                            not hasattr(child, 'default_multi') and \
                            (not is_leadership or leader_is_mandatory):
                        if not hasattr(child, 'test'):
                            raise Exception(f'{child.path} need test tag to document')
                        value = child.test
                        if ((not is_leadership or not idx) and child.multi is True) or (is_leadership and idx and child.multi == 'submulti'):
                            value = [value]
                        if is_leadership and idx:
                            value = {'0': value}
                        mandatory_values_without_values[child.path] = value
                    parameter = add_parameter_variable(idx, child, is_leadership, 'variable', sub_dynamic_path)
                    #
                    if child.name == child.doc:
                        raise Exception(f'missing doc for variable {child.path} in {child.xmlfiles}')
                    description = add_title_variable(child, is_leadership, is_dynamic, objectspace)
                    values = {'parameter': parameter,
                              'description': description,
                              'type': 'option'}
                    if hasattr(child, 'supplier') and child.supplier:
                        values['supplier'] = child.supplier
                    add_mode(child, values, modes, elt, is_leadership)
                    dico[child.path] = values
                elif is_service:
                    if hasattr(child, 'provider'):
                        provider = child.provider
                        if ':' not in provider:
                            providers_suppliers['providers'].setdefault(provider, []).append(applicationservice)
                    if hasattr(child, 'supplier') and child.supplier:
                        supplier = child.supplier
                        if ':' not in supplier:
                            providers_suppliers['suppliers'].setdefault(supplier, []).append(applicationservice)
                            if 'mandatory' in child.properties:
                                providers_suppliers['mandatory_suppliers'].setdefault(supplier, []).append(applicationservice)
            else:
                parse(applicationservice,
                      elts + [child],
                      dico,
                      providers_suppliers,
                      hidden,
                      dynamic,
                      objectspace,
                      mandatory_values_without_values,
                      modes,
                      is_dynamic,
                      dynamic_path,
                      is_service,
                      suppliers_variables,
                      )
    return first_variable


def build_dependencies_tree(applicationservice,
                            applicationservice_data,
                            applicationservices_data,
                            applicationservices_data_ext,
                            space,
                            only_name: bool=False,
                            ):
    depends = []
    if applicationservice_data['depends']:
        if applicationservice in applicationservices_data:
            app_data = applicationservices_data[applicationservice]
        else:
            for url, apps_data in applicationservices_data_ext.items():
                if applicationservice in apps_data:
                    app_data = apps_data[applicationservice]
                    break
            else:
                raise Exception(f'cannot find applicationservice "{applicationservice}"')
        for idx, depend in enumerate(app_data['depends']):
            if depend in applicationservices_data:
                url = '..'
                ext = False
            else:
                for url, apps_data in applicationservices_data_ext.items():
                    if depend in apps_data:
                        break
                else:
                    raise Exception(f'cannot find applicationservice "{applicationservice}"')
                ext = True
            subdepends = build_dependencies_tree(depend,
                                                 applicationservice_data,
                                                 applicationservices_data,
                                                 applicationservices_data_ext,
                                                 space + 2,
                                                 only_name,
                                                 )
            if only_name:
                if depend not in depends:
                    depends.append(depend)
                for dep in subdepends:
                    if dep not in depends:
                        depends.append(dep)
            else:
                if not idx or subdepends:
                    title = '\n'
                else:
                    title = ''
                depend_desc = depend
                if ext:
                    depend_desc += ' (in external dataset)'
                doc = applicationservices_data[depend]["description"]
                if not doc.endswith('.'):
                    doc += '.'
                title = ' ' * space + f'- [{depend_desc}]({url}/{depend}/README.md): {doc}'
                depends.append(title)
                depends.extend(subdepends)
    return depends


def load_data(url, directory, applicationservices_data, global_data={}):
    root_path = join(directory, 'seed')
    applicationservices = listdir(root_path)
    tmps = {}
    for applicationservice in applicationservices:
        as_dir = join(root_path, applicationservice)
        if not isdir(as_dir):
            continue
        applicationservice_data = load_application_service(as_dir)
        if not applicationservice_data.get('documentation', True):
            continue
        applicationservices_data[applicationservice] = {'description': applicationservice_data['description'],
                                                        'website': applicationservice_data.get('website'),
                                                        'help': applicationservice_data.get('help'),
                                                        'as_dir': as_dir,
                                                        'depends': [],
                                                        'used_by': [],
                                                        'service': applicationservice_data.get('service', False),
                                                        }
        if applicationservice in tmps:
            for app in tmps.pop(applicationservice):
                used_by = f'[{app}](../{app}/README.md)'
                applicationservices_data[applicationservice]['used_by'].append(used_by)
        if 'depends' in applicationservice_data:
            for depend in applicationservice_data['depends']:
                applicationservices_data[applicationservice]['depends'].append(depend)
                if depend in applicationservices_data:
                    used_by = f'[{applicationservice}](../{applicationservice}/README.md)'
                    applicationservices_data[depend]['used_by'].append(used_by)
                else:
                    tmps.setdefault(depend, []).append(applicationservice)
    if tmps and global_data:
        for depend, applications in tmps.items():
            for app in applications:
                used_by = f'[{app} (in external dataset)]({url}/{app}/README.md)'
                global_data[depend]['used_by'].append(used_by)


def display_example(as_fh,
                    applicationservice,
                    provider_zone,
                    zones_name,
                    values,
                    ):
    as_fh.write('\n## Example\n')
    as_fh.write('\nZone names are provided as examples. Think about adapting with the value of provider_zone in configuration file.\n')
    as_fh.write('\n```\n')
    example = {applicationservice: {'applicationservice': applicationservice}}
    if provider_zone:
        example[applicationservice]['provider_zone'] = provider_zone
    if zones_name:
        example[applicationservice]['zones_name'] = zones_name
    if values:
        example[applicationservice]['values'] = values
#    as_fh.write(dump(example, sort_keys=False))
    yaml = YAML()
    yaml.indent(sequence=4, offset=2)
    yaml.dump(example, as_fh)
    as_fh.write('```\n')


def load_dict(applicationservices,
              applicationservices_data,
              suppliers_variables,
              dico,
              providers_suppliers,
              mandatory_values_without_values,
              just_doc=True,
              ):
    rougailconfig['dictionaries_dir'] = []
    extra_dictionaries = {}
    functions_file = [func.__file__]
    for applicationservice in applicationservices:
        as_dir = applicationservices_data[applicationservice]['as_dir']
        dirname = join(as_dir, 'dictionaries')
        if isdir(dirname):
            rougailconfig['dictionaries_dir'].append(dirname)
        dirname_extras = join(as_dir, 'extras')
        if isdir(dirname_extras):
            for extra in listdir(dirname_extras):
                extra_dir = join(dirname_extras, extra)
                if isdir(extra_dir):
                    extra_dictionaries.setdefault(extra, []).append(extra_dir)
        dirname_func = join(as_dir, 'funcs')
        if isdir(dirname_func):
            for f in listdir(dirname_func):
                if f.startswith('__'):
                    continue
                functions_file.append(abspath(join(dirname_func, f)))
    rougailconfig['extra_dictionaries'] = extra_dictionaries
    rougailconfig['functions_file'] = functions_file
    rougailconfig['extra_annotators'] = EXTRA_ANNOTATORS
    converted = RougailConvert(rougailconfig, just_doc=just_doc)
    converted.load_dictionaries()
    converted.annotate()
    objectspace = converted.rougailobjspace
    modes = {name: Mode(idx) for idx, name in enumerate(objectspace.rougailconfig['modes_level'])}
    if hasattr(objectspace.space, 'variables'):
        dico[applicationservice] = {}
        if mandatory_values_without_values is not None:
            mandatory_values_without_values[applicationservice] = {}
            mandatory_values = mandatory_values_without_values[applicationservice]
        else:
            mandatory_values = None
        for name, elt in objectspace.space.variables.items():
            parse(applicationservice,
                  [elt],
                  dico[applicationservice],
                  providers_suppliers,
                  False,
                  False,
                  objectspace,
                  mandatory_values,
                  modes,
                  False,
                  '',
                  applicationservices_data[applicationservice]['service'],
                  suppliers_variables,
                  )


def write_data(applicationservices_data, applicationservices_data_ext):
    dico = {}
    providers_suppliers = {'providers': {}, 'suppliers': {}, 'mandatory_suppliers': {}, 'depends': {}}
    mandatory_values_without_values = {}
    suppliers_variables = {}
    for applicationservice, applicationservice_data in applicationservices_data.items():
        if applicationservice_data['service']:
            depends = build_dependencies_tree(applicationservice,
                                              applicationservice_data,
                                              applicationservices_data,
                                              applicationservices_data_ext,
                                              0,
                                              only_name=True,
                                              )
            depends.append(applicationservice)
            providers_suppliers['depends'][applicationservice] = set(depends)
            load_dict(['provider-systemd-machined'] + depends,
                      applicationservices_data,
                      suppliers_variables,
                      dico,
                      providers_suppliers,
                      mandatory_values_without_values,
                      just_doc=False,
                      )
        else:
            load_dict([applicationservice],
                      applicationservices_data,
                      suppliers_variables,
                      dico,
                      providers_suppliers,
                      None,
                      )
    for provider in suppliers_variables:
        for supplier, tdico in suppliers_variables[provider].items():
            if supplier == 'doc':
                continue
            if not tdico['description']:
                raise Exception(f'variable with provider "{supplier}" must have a description')
    #
    for filename in glob('seed/README.*.md'):
        unlink(filename)
    for provider in suppliers_variables:
        with open(join(f'seed/README.{provider}.md'), 'w') as as_fh:
            as_fh.write(f'---\ngitea: none\ninclude_toc: true\n---\n\n')
            as_fh.write(f'\n[Return to the list of application services.](README.md)\n')
            as_fh.write(f'# {provider}\n\n')
            as_fh.write(f'## Synopsis\n\n')
            as_fh.write(suppliers_variables[provider][provider]['description'])
            as_fh.write(f'\n\n## Variables\n\n')
            providers = []
            for provider_name, tdico in suppliers_variables[provider].items():
                if ':' in provider_name:
                    providers.append(list(tdico.values()))
            if providers:
                as_fh.write(tabulate(providers, headers=['Parameter', 'Comment'], tablefmt="github"))
                as_fh.write(f'\n')
            applicationservices = providers_suppliers['providers'].get(provider, [])
            if applicationservices:
                if len(applicationservices) == 1:
                    as_fh.write(f'\n## Provider\n\n')
                    for applicationservice in applicationservices:
                        doc = applicationservices_data[applicationservice]["description"]
                        if not doc.endswith('.'):
                            doc += '.'
                        as_fh.write(f'[{applicationservice}]({applicationservice}/README.md): {doc}\n')
                else:
                    as_fh.write(f'\n## Providers\n\n')
                    for applicationservice in applicationservices:
                        doc = applicationservices_data[applicationservice]["description"]
                        if not doc.endswith('.'):
                            doc += '.'
                        as_fh.write(f'- [{applicationservice}]({applicationservice}/README.md): {doc}\n')
            applicationservices = providers_suppliers['suppliers'].get(provider, [])
            if applicationservices:
                if len(applicationservices) == 1:
                    as_fh.write(f'\n## Supplier\n\n')
                    for applicationservice in applicationservices:
                        doc = applicationservices_data[applicationservice]["description"]
                        if not doc.endswith('.'):
                            doc += '.'
                        as_fh.write(f'[{applicationservice}]({applicationservice}/README.md): {doc}\n')
                else:
                    as_fh.write(f'\n## Suppliers\n\n')
                    for applicationservice in applicationservices:
                        doc = applicationservices_data[applicationservice]["description"]
                        if not doc.endswith('.'):
                            doc += '.'
                        as_fh.write(f'- [{applicationservice}]({applicationservice}/README.md): {doc}\n')
    #
    for applicationservice, applicationservice_data in applicationservices_data.items():
        as_dir = applicationservice_data['as_dir']
        local_dico = dico
        with open(join(as_dir, 'README.md'), 'w') as as_fh:
            as_fh.write(f'---\ngitea: none\ninclude_toc: true\n---\n\n')
            as_fh.write(f'\n[Return to the list of application services.](../README.md)\n')
            as_fh.write(f'# {applicationservice}\n\n')
            as_fh.write(f'## Synopsis\n\n')
            description = applicationservice_data['description'] + '.'
            if applicationservice_data['website']:
                description = f"\n[{description}]({applicationservice_data['website']})"
            description += '\n'
            if applicationservice_data['help']:
                help_ = applicationservice_data['help'].strip()
                if not help_.endswith('.'):
                    help_ += 'p.'
                description += f'\n{help_}\n'
            as_fh.write(description)
            # variables
            if applicationservice in local_dico and local_dico[applicationservice]:
                display_variables(as_dir, as_fh, applicationservice, local_dico)
            if applicationservice_data['service']:
                mandatory_zones_name = {}
                zones_name = {}
                provider_name = None
                if applicationservice_data['depends']:
                    depends_set = providers_suppliers['depends'][applicationservice]
                    # providers
                    for provider, provider_as in providers_suppliers['providers'].items():
                        if not set(provider_as) & depends_set:
                            continue
                        provider_name = provider.lower()
                        break
                    #
                    if providers_suppliers['suppliers'] and list(providers_suppliers['suppliers']) != ['Host']:
                        as_fh.write('\n## Requirements services\n')
                        provider_hosts = []
                        for provider, provider_data in providers_suppliers['suppliers'].items():
                            if provider == 'Host':
                                continue
                            if applicationservice not in provider_data:
                                continue
                            suppliers = providers_suppliers['providers'].get(provider, [])
                            if len(suppliers) == 1:
                                if suppliers[0] in provider_hosts:
                                    continue
                                provider_hosts.append(suppliers[0])
                            filename = f'README.{provider}.md'
                            if isfile(join('seed', filename)):
                                msg = f'[{provider}](../{filename})'
                            else:
                                msg = provider
                            if provider in providers_suppliers['mandatory_suppliers']:
                                mandatory_zones_name[provider] = msg
                            else:
                                zones_name[provider] = msg
                        if mandatory_zones_name:
                            as_fh.write('\n### Mandatories\n\n')
                            for provider, msg in mandatory_zones_name.items():
                                as_fh.write(f'- {msg}: {suppliers_variables[provider]["doc"]}\n')
                        if zones_name:
                            as_fh.write('\n### Optionals\n\n')
                            for provider, msg in zones_name.items():
                                as_fh.write(f'- {msg}: {suppliers_variables[provider]["doc"]}\n')
                    #
                    mandatory_zones_name = [provider.lower() for provider in mandatory_zones_name]
                    mandatory_zones_name.sort()
                    display_example(as_fh,
                                    applicationservice,
                                    provider_name,
                                    mandatory_zones_name,
                                    mandatory_values_without_values[applicationservice],
                                    )
            if applicationservice_data['depends']:
                as_fh.write('\n## Dependances\n\n')
                for depend in build_dependencies_tree(applicationservice, applicationservice_data, applicationservices_data, applicationservices_data_ext, 0):
                    as_fh.write(f'{depend}\n')
            if applicationservice_data['used_by']:
                as_fh.write('\n## Used by\n\n')
                if len(applicationservice_data['used_by']) == 1:
                    link = applicationservice_data['used_by'][0]
                    as_fh.write(f'{link}\n')
                else:
                    for link in applicationservice_data['used_by']:
                        as_fh.write(f'- {link}\n')
            linked = []
            for provider, provider_as in providers_suppliers['providers'].items():
                if not applicationservice in provider_as:
                    continue
                if provider in providers_suppliers['suppliers']:
                    for supplier in providers_suppliers['suppliers'][provider]:
                        if supplier in linked:
                            continue
                        linked.append(supplier)
            linked.sort()
            if linked:
                if len(linked) == 1:
                    as_fh.write('\n## Useful for service\n\n')
                    description = applicationservices_data[linked[0]]["description"]
                    if not description.endswith('.'):
                        description += '.'
                    as_fh.write(f'[{linked[0]}](../{linked[0]}/README.md): {description}\n')
                else:
                    as_fh.write('\n## Useful for services\n\n')
                    for supplier in linked:
                        description = applicationservices_data[supplier]["description"]
                        if not description.endswith('.'):
                            description += '.'
                        as_fh.write(f'- [{supplier}](../{supplier}/README.md): {description}\n')

    with open('seed/README.md', 'w') as as_fh:
        for service in [True, False]:
            if service:
                as_fh.write('# Application services\n\n')
            else:
                as_fh.write('# Application dependencies\n\n')
            applicationservices = {}
            for applicationservice, applicationservice_data in applicationservices_data.items():
                if applicationservice_data['service'] is not service:
                    continue
                applicationservices.setdefault(applicationservice.split('-')[0], []).append(applicationservice)
            applicationservice_categories = list(applicationservices.keys())
            applicationservice_categories.sort()
            for category in applicationservice_categories:
                applicationservices_ = applicationservices[category]
                if len(applicationservices_) == 1:
                    applicationservice = applicationservices_[0]
                    applicationservice_data = applicationservices_data[applicationservice]
                    as_fh.write(f'- [{applicationservice}]({applicationservice}/README.md): {applicationservice_data["description"]}\n')
                else:
                    as_fh.write(f'- {category}:\n')
                    applicationservices_.sort()
                    for applicationservice in applicationservices_:
                        applicationservice_data = applicationservices_data[applicationservice]
                        as_fh.write(f'  - [{applicationservice}]({applicationservice}/README.md): {applicationservice_data["description"]}\n')
        providers = list(providers_suppliers['providers'].keys())
        providers.sort()
        if providers:
            as_fh.write('\n# Providers\n\n')
        for provider in providers:
            if provider in suppliers_variables[provider]:
                as_fh.write(f'- [{provider}](README.{provider}.md): {suppliers_variables[provider]["doc"]}\n')


def display_variables(as_dir, as_fh, applicationservice, local_dico):
    for main_title, modes in [('Basic variables', ['basic']),
                              ('Variables', ['basic', 'normal']),
                              ('Variables for expert', ['basic', 'normal', 'expert']),
                              ]:
        main_title_added = False
        titles = []
        variables = []
        parent_path = None
        for title, data in list(local_dico[applicationservice].items()):
            keys = []
            if data['type'] == 'optiondescription':
                #if data['mode'] not in modes:
                #    continue
                if variables:
                    as_fh.write(tabulate(variables, headers=['Parameter', 'Comment'], tablefmt="github"))
                    as_fh.write('\n')
                    as_fh.write('\n')
                    variables = []
                    parent_path = None
                title_level = data['level']
                len_titles = len(titles)
                if len_titles < title_level:
                    titles.append((None, None))
                titles[title_level - 1] = (title, data)
                if len_titles > title_level:
                    for idx in range(title_level, len_titles):
                        titles.pop(title_level)
            else:
                if data['mode'] not in modes:
                    continue
                if not main_title_added:
                    as_fh.write(f'\n## {main_title}\n\n')
                    main_title_added = True
                for idx, td in enumerate(titles):
                    t, d = td
                    if not t:
                        continue
                    as_fh.write(f'{t}\n\n')
                    if d['help']:
                        help = d['help']
                        if not help.endswith('.'):
                            help += '.'
                        as_fh.write(f'{help}\n\n')
                    titles[idx] = (None, None)
                if data['mode'] != modes[-1]:
                    continue
                new_parent_path = title.rsplit('.', 1)[0]
                if variables and parent_path != new_parent_path:
                    as_fh.write(tabulate(variables, headers=['Parameter', 'Comments'], tablefmt="github"))
                    as_fh.write('\n')
                    as_fh.write('\n')
                    variables = []
                parent_path = new_parent_path
                variables.append([data['parameter'], data['description']])
                del local_dico[applicationservice][title]
        if variables:
            as_fh.write(tabulate(variables, headers=['Parameter', 'Comments'], tablefmt="github"))
        as_fh.write('\n')


def main():
    applicationservices_data = {}
    load_data('..', '', applicationservices_data)
    applicationservices_data_ext = {}
    for arg in argv[1:]:
        if '=' not in arg:
            raise Exception(f'cannot parse argument "{arg}", should be dataset_path=url')
        path, url = arg.split('=', 1)
        if url in applicationservices_data_ext:
            raise Exception(f'duplicate url "{url}" in arguments')
        applicationservices_data_ext[url] = {}
        load_data(url, path, applicationservices_data_ext[url], applicationservices_data)
    write_data(applicationservices_data, applicationservices_data_ext)


if __name__ == '__main__':
    main()