add risotto_auto_doc script

sbin/risotto_auto_doc

@@ -0,0 +1,277 @@
+#!/usr/bin/env python3
+from os import listdir
+from os.path import isdir, join
+from tabulate import tabulate
+
+from rougail import RougailConfig
+from rougail.convert import RougailConvert
+from rougail.objspace import RootRougailObject
+from risotto.utils import EXTRA_ANNOTATORS, ROUGAIL_NAMESPACE, ROUGAIL_NAMESPACE_DESCRIPTION
+from risotto.image import load_application_service
+
+
+rougailconfig = RougailConfig
+rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE
+rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
+
+
+DEFAULT_TYPE = 'string'
+ROUGAIL_VARIABLE_TYPE = 'https://cloud.silique.fr/gitea/risotto/rougail/src/branch/main/doc/variable/README.md#le-type-de-la-variable'
+
+
+def add_title_family(elts, dico):
+    for idx, elt in enumerate(elts):
+        description = elt.doc
+        if not idx:
+            description = description.capitalize()
+        space = idx + 3
+        title = '#' * space + f' {description} (*{elt.path}*)'
+        if title not in dico:
+            dico[title] = {'variables': [], 'help': '', 'type': ''}
+            if hasattr(elt, 'information') and hasattr(elt.information, 'help'):
+                dico[title]['help'] = elt.information.help
+        if hasattr(elt, 'suffixes') and elt.suffixes:
+            dico[title]['type'] = 'dynamic'
+            dico[title]['suffixes'] = elt.suffixes.path
+        if hasattr(elt, 'leadership') and elt.leadership:
+            dico[title]['type'] = 'leadership'
+    return title
+
+
+
+def parse(applicationservice, elts, dico, providers_suppliers, hidden):
+    elt = elts[-1]
+    first_variable = True
+    if not hidden:
+        hidden = hasattr(elt, 'properties') and ('hidden' in elt.properties or 'disabled' in elt.properties)
+    is_leadership = hasattr(elt, 'leadership') and elt.leadership
+    for children in vars(elt).values():
+        if isinstance(children, dict):
+            children = list(children.values())
+        if not isinstance(children, list):
+            continue
+        for idx, child in enumerate(children):
+            if isinstance(child, objectspace.property_) or \
+                    not isinstance(child, RootRougailObject):
+                continue
+            if isinstance(child, objectspace.variable):
+                if not hidden and (not hasattr(child, 'properties') or ('hidden' not in child.properties and not 'disabled' in child.properties)):
+                    if first_variable:
+                        title = add_title_family(elts, dico)
+                        first_variable = False
+                    var_title = child.doc
+                    if hasattr(child, 'properties') and 'mandatory' in child.properties:
+                        var_title = '**' + var_title + '**'
+                    var_path = child.xmlfiles[-1].split('/', 2)[-1]
+                    if child.doc != child.name:
+                        var_title += f' (*[{child.name}]({var_path})*)'
+                    else:
+                        var_title = f'*[{var_title}]({var_path})*'
+                    if ((idx == 0 or not is_leadership) and child.multi is True) or (idx != 0 and is_leadership and child.multi == 'submulti'):
+                        var_title += ' [+]'
+                    values = {'description': var_title,
+                              }
+                    if hasattr(child, 'information') and hasattr(child.information, 'help'):
+                        values['help'] = child.information.help
+                    if child.type != DEFAULT_TYPE:
+                        values['type'] = child.type
+                    if hasattr(child, 'default'):
+                        default = child.default
+                        if isinstance(default, list):
+                            default = '<br />'.join(default)
+                        values['values'] = default
+                    if hasattr(child, 'choice'):
+                        values['choices'] = '<br />'.join([choice.name for choice in child.choice])
+                    if hasattr(child, 'provider'):
+                        provider = child.provider
+                        values['provider'] = provider
+                        if ':' not in provider:
+                            providers_suppliers['providers'].setdefault(provider, []).append(applicationservice)
+                    if hasattr(child, 'supplier'):
+                        supplier = child.supplier
+                        values['supplier'] = supplier
+                        if ':' not in supplier:
+                            providers_suppliers['suppliers'].setdefault(supplier, []).append(applicationservice)
+                    dico[title]['variables'].append(values)
+                else:
+                    if hasattr(child, 'provider'):
+                        provider = child.provider
+                        if ':' not in provider:
+                            providers_suppliers['providers'].setdefault(provider, []).append(applicationservice)
+                    if hasattr(child, 'supplier'):
+                        supplier = child.supplier
+                        if ':' not in supplier:
+                            providers_suppliers['suppliers'].setdefault(supplier, []).append(applicationservice)
+            else:
+                parse(applicationservice, elts + [child], dico, providers_suppliers, hidden)
+
+
+applicationservices = listdir('seed')
+#applicationservices = ['speedtest-rs']
+#applicationservices = ['redis']
+applicationservices_data = {}
+tmps = {}
+for applicationservice in applicationservices:
+    as_dir = join('seed', applicationservice)
+    if not isdir(as_dir):
+        continue
+    applicationservice_data = load_application_service(as_dir)
+    applicationservices_data[applicationservice] = {'description': applicationservice_data['description'],
+                                                    'website': applicationservice_data.get('website'),
+                                                    'as_dir': as_dir,
+                                                    'depends': [],
+                                                    'used_by': [],
+                                                    }
+    if applicationservice in tmps:
+        applicationservices_data[applicationservice]['used_by'] = tmps.pop(applicationservice)
+    if 'depends' in applicationservice_data:
+        for depend in applicationservice_data['depends']:
+            applicationservices_data[applicationservice]['depends'].append(depend)
+            if depend in applicationservices_data:
+                applicationservices_data[depend]['used_by'].append(applicationservice)
+            else:
+                tmps.setdefault(depend, []).append(applicationservice)
+
+dico = {}
+providers_suppliers = {'providers': {}, 'suppliers': {}}
+for applicationservice, applicationservice_data in applicationservices_data.items():
+    as_dir = applicationservice_data['as_dir']
+    dirname = join(as_dir, 'dictionaries')
+    if isdir(dirname):
+        rougailconfig['dictionaries_dir'] = [dirname]
+    else:
+        rougailconfig['dictionaries_dir'] = []
+    dirname_extras = join(as_dir, 'extras')
+    extra_dictionaries = {}
+    if isdir(dirname_extras):
+        for extra in listdir(dirname_extras):
+            extra_dir = join(dirname_extras, extra)
+            if isdir(extra_dir):
+                extra_dictionaries.setdefault(extra, []).append(extra_dir)
+    if not isdir(dirname) and not extra_dictionaries:
+        continue
+    rougailconfig['extra_dictionaries'] = extra_dictionaries
+    converted = RougailConvert(rougailconfig)
+    converted.load_dictionaries(just_doc=True)
+    converted.annotate()
+    objectspace = converted.rougailobjspace
+    if hasattr(objectspace.space, 'variables'):
+        dico[applicationservice] = {}
+        for name, elt in objectspace.space.variables.items():
+            parse(applicationservice, [elt], dico[applicationservice], providers_suppliers, False)
+
+
+def build_dependencies_tree(applicationservice, space):
+    depends = []
+    if applicationservice_data['depends']:
+        for idx, depend in enumerate(applicationservices_data[applicationservice]['depends']):
+            subdepends = build_dependencies_tree(depend, space + 2)
+            if not idx or subdepends:
+                title = '\n'
+            else:
+                title = ''
+            title = ' ' * space + f'- [{depend}](../{depend}/README.md)'
+            depends.append(title)
+            depends.extend(subdepends)
+    return depends
+
+
+for applicationservice, applicationservice_data in applicationservices_data.items():
+    as_dir = applicationservice_data['as_dir']
+    with open(join(as_dir, 'README.md'), 'w') as as_fh:
+        as_fh.write(f'---\ngitea: none\ninclude_toc: true\n---\n\n')
+        as_fh.write(f'# {applicationservice}\n\n')
+        as_fh.write(f'[All applications services for this dataset.](../README.md)\n\n')
+        as_fh.write(f'## Description\n\n')
+        description = applicationservice_data['description'] + '.\n'
+        if applicationservice_data['website']:
+            description += f'\n[For more informations]({applicationservice_data["website"]})\n'
+        as_fh.write(description)
+        if applicationservice_data['depends']:
+            as_fh.write(f'\n## Dependances\n\n')
+            for depend in build_dependencies_tree(applicationservice, 0):
+                as_fh.write(f'{depend}\n')
+        if applicationservice in dico and dico[applicationservice]:
+            as_fh.write('\n## Variables\n\n')
+            for title, data in dico[applicationservice].items():
+                as_fh.write(f'{title}\n')
+                if data['type'] == 'leadership':
+                    as_fh.write('\nThis a family is a leadership.\n')
+                if data['type'] == 'dynamic':
+                    as_fh.write(f'\nThis a dynamic family generated from the variable "{data["suffixes"]}".\n')
+                if data['help']:
+                    as_fh.write(f'\n{data["help"]}\n')
+                keys = []
+                if data['variables']:
+                    variables = data['variables']
+                    for variable in variables:
+                        for key in variable:
+                            if key not in keys:
+                                keys.append(key)
+                    values = []
+                    for variable in variables:
+                        value = []
+                        for key in keys:
+                            if key in variable:
+                                val = variable[key]
+                            elif key == 'type':
+                                val = DEFAULT_TYPE
+                            else:
+                                val = ''
+                            if key == 'type':
+                                val = f'[{val}]({ROUGAIL_VARIABLE_TYPE})'
+                            value.append(val)
+                        values.append(value)
+                    as_fh.write('\n')
+                    as_fh.write(tabulate(values, headers=[key.capitalize() for key in keys], tablefmt="github"))
+                    as_fh.write('\n')
+                as_fh.write('\n')
+            # FIXME if not applicationservice_data['used_by']:
+            # FIXME     as_fh.write('\n## Variables with dependencies\n\n')
+            as_fh.write('\n- [+]: variable is multiple\n- **bold**: variable is mandatory\n')
+        if applicationservice_data['used_by']:
+            as_fh.write('\n## Used by\n\n')
+            for link in applicationservice_data['used_by']:
+                as_fh.write(f'- [{link}](../{link}/README.md)\n')
+        linked = []
+        for provider, provider_as in providers_suppliers['providers'].items():
+            if not applicationservice in provider_as:
+                continue
+            for supplier in providers_suppliers['suppliers'][provider]:
+                if not linked:
+                    as_fh.write('\n## Linked to\n\n')
+                if supplier in linked:
+                    continue
+                as_fh.write(f'- [{supplier}](../{supplier}/README.md)\n')
+                linked.append(supplier)
+        for supplier, supplier_as in providers_suppliers['suppliers'].items():
+            if not applicationservice in supplier_as:
+                continue
+            for provider in providers_suppliers['providers'][supplier]:
+                if not linked:
+                    as_fh.write('\n## Linked to\n\n')
+                if provider in linked:
+                    continue
+                as_fh.write(f'- [{provider}](../{provider}/README.md)\n')
+                linked.append(provider)
+
+
+with open('seed/README.md', 'w') as as_fh:
+    as_fh.write('# Application services\n\n')
+    applicationservices = {}
+    for applicationservice in applicationservices_data:
+        applicationservices.setdefault(applicationservice.split('-')[0], []).append(applicationservice)
+    applicationservice_categories = list(applicationservices.keys())
+    applicationservice_categories.sort()
+    for category in applicationservice_categories:
+        applicationservices_ = applicationservices[category]
+        if len(applicationservices_) == 1:
+            applicationservice = applicationservices_[0]
+            applicationservice_data = applicationservices_data[applicationservice]
+            as_fh.write(f'- [{applicationservice}]({applicationservice}/README.md): {applicationservice_data["description"]}\n')
+        else:
+            as_fh.write(f'- {category}:\n')
+            applicationservices_.sort()
+            for applicationservice in applicationservices_:
+                applicationservice_data = applicationservices_data[applicationservice]
+                as_fh.write(f'  - [{applicationservice}]({applicationservice}/README.md): {applicationservice_data["description"]}\n')

src/risotto/image.py

@@ -24,6 +24,9 @@ class ModuleCfg():
     def __repr__(self):
         return str(vars(self))
 
+def load_application_service(as_dir: str) -> str:
+    with open(join(as_dir, 'applicationservice.yml')) as yaml:
+        return yaml_load(yaml, Loader=SafeLoader)
 
 class Applications:
     def __init__(self) -> None:
@@ -119,8 +122,7 @@ class Modules:
         self._load_applicationservice_directories(as_dir,
                                                   cfg,
                                                   )
-        with open(join(as_dir, 'applicationservice.yml')) as yaml:
+        app = load_application_service(as_dir)
-            app = yaml_load(yaml, Loader=SafeLoader)
         provider = app.get('provider')
         if provider:
             cfg.providers.setdefault(provider, [])

src/risotto/machine.py

@@ -1,4 +1,4 @@
-from .utils import MULTI_FUNCTIONS, load_zones, value_pprint, RISOTTO_CONFIG
+from .utils import MULTI_FUNCTIONS, load_zones, value_pprint, RISOTTO_CONFIG, EXTRA_ANNOTATORS, ROUGAIL_NAMESPACE, ROUGAIL_NAMESPACE_DESCRIPTION
 from .image import Applications, Modules, valid_mandatories, applicationservice_copy
 from .rougail.annotator import calc_providers, calc_providers_global, calc_providers_dynamic, calc_providers_dynamic_follower, calc_providers_follower
 
@@ -27,8 +27,6 @@ def tiramisu_display_name(kls,
 
 
 CONFIG_FILE = 'servers.yml'
-ROUGAIL_NAMESPACE = 'general'
-ROUGAIL_NAMESPACE_DESCRIPTION = 'Général'
 TIRAMISU_CACHE = 'tiramisu_cache.py'
 VALUES_CACHE = 'values_cache.json'
 INFORMATIONS_CACHE = 'informations_cache.json'
@@ -148,7 +146,7 @@ class Loader:
         cfg['variable_namespace'] = ROUGAIL_NAMESPACE
         cfg['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
         cfg['multi_functions'] = MULTI_FUNCTIONS
-        cfg['extra_annotators'] = ['risotto.rougail']
+        cfg['extra_annotators'] = EXTRA_ANNOTATORS
         cfg['internal_functions'] = list(FUNCTIONS.keys())
         cfg['force_convert_dyn_option_description'] = True
         cfg['risotto_globals'] = {}

src/risotto/utils.py

@@ -1,4 +1,5 @@
 from os import  environ
+from os.path import isfile
 from typing import List
 from ipaddress import ip_address
 from toml import load as toml_load
@@ -6,10 +7,17 @@ from pprint import pprint
 
 
 MULTI_FUNCTIONS = []
+EXTRA_ANNOTATORS = ['risotto.rougail']
+ROUGAIL_NAMESPACE = 'general'
+ROUGAIL_NAMESPACE_DESCRIPTION = 'Général'
 
 
-with open(environ.get('CONFIG_FILE', 'risotto.conf'), 'r') as fh:
+config_file = environ.get('CONFIG_FILE', 'risotto.conf')
+if isfile(config_file):
+    with open(config_file, 'r') as fh:
         RISOTTO_CONFIG = toml_load(fh)
+else:
+    RISOTTO_CONFIG = {}
 
 
 def _(s):