From 124c6b56d2e8740a70fc778d0c69113b276316a0 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Sun, 25 Dec 2022 17:21:03 +0100 Subject: [PATCH] add risotto_auto_doc script --- sbin/risotto_auto_doc | 277 +++++++++++++++++++++++++++++++++++++++++ src/risotto/image.py | 6 +- src/risotto/machine.py | 6 +- src/risotto/utils.py | 12 +- 4 files changed, 293 insertions(+), 8 deletions(-) create mode 100755 sbin/risotto_auto_doc diff --git a/sbin/risotto_auto_doc b/sbin/risotto_auto_doc new file mode 100755 index 0000000..cf57604 --- /dev/null +++ b/sbin/risotto_auto_doc @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +from os import listdir +from os.path import isdir, join +from tabulate import tabulate + +from rougail import RougailConfig +from rougail.convert import RougailConvert +from rougail.objspace import RootRougailObject +from risotto.utils import EXTRA_ANNOTATORS, ROUGAIL_NAMESPACE, ROUGAIL_NAMESPACE_DESCRIPTION +from risotto.image import load_application_service + + +rougailconfig = RougailConfig +rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE +rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION + + +DEFAULT_TYPE = 'string' +ROUGAIL_VARIABLE_TYPE = 'https://cloud.silique.fr/gitea/risotto/rougail/src/branch/main/doc/variable/README.md#le-type-de-la-variable' + + +def add_title_family(elts, dico): + for idx, elt in enumerate(elts): + description = elt.doc + if not idx: + description = description.capitalize() + space = idx + 3 + title = '#' * space + f' {description} (*{elt.path}*)' + if title not in dico: + dico[title] = {'variables': [], 'help': '', 'type': ''} + if hasattr(elt, 'information') and hasattr(elt.information, 'help'): + dico[title]['help'] = elt.information.help + if hasattr(elt, 'suffixes') and elt.suffixes: + dico[title]['type'] = 'dynamic' + dico[title]['suffixes'] = elt.suffixes.path + if hasattr(elt, 'leadership') and elt.leadership: + dico[title]['type'] = 'leadership' + return title + + + +def parse(applicationservice, elts, dico, providers_suppliers, hidden): + elt = elts[-1] + first_variable = True + if not hidden: + hidden = hasattr(elt, 'properties') and ('hidden' in elt.properties or 'disabled' in elt.properties) + is_leadership = hasattr(elt, 'leadership') and elt.leadership + for children in vars(elt).values(): + if isinstance(children, dict): + children = list(children.values()) + if not isinstance(children, list): + continue + for idx, child in enumerate(children): + if isinstance(child, objectspace.property_) or \ + not isinstance(child, RootRougailObject): + continue + if isinstance(child, objectspace.variable): + if not hidden and (not hasattr(child, 'properties') or ('hidden' not in child.properties and not 'disabled' in child.properties)): + if first_variable: + title = add_title_family(elts, dico) + first_variable = False + var_title = child.doc + if hasattr(child, 'properties') and 'mandatory' in child.properties: + var_title = '**' + var_title + '**' + var_path = child.xmlfiles[-1].split('/', 2)[-1] + if child.doc != child.name: + var_title += f' (*[{child.name}]({var_path})*)' + else: + var_title = f'*[{var_title}]({var_path})*' + if ((idx == 0 or not is_leadership) and child.multi is True) or (idx != 0 and is_leadership and child.multi == 'submulti'): + var_title += ' [+]' + values = {'description': var_title, + } + if hasattr(child, 'information') and hasattr(child.information, 'help'): + values['help'] = child.information.help + if child.type != DEFAULT_TYPE: + values['type'] = child.type + if hasattr(child, 'default'): + default = child.default + if isinstance(default, list): + default = '
'.join(default) + values['values'] = default + if hasattr(child, 'choice'): + values['choices'] = '
'.join([choice.name for choice in child.choice]) + if hasattr(child, 'provider'): + provider = child.provider + values['provider'] = provider + if ':' not in provider: + providers_suppliers['providers'].setdefault(provider, []).append(applicationservice) + if hasattr(child, 'supplier'): + supplier = child.supplier + values['supplier'] = supplier + if ':' not in supplier: + providers_suppliers['suppliers'].setdefault(supplier, []).append(applicationservice) + dico[title]['variables'].append(values) + else: + if hasattr(child, 'provider'): + provider = child.provider + if ':' not in provider: + providers_suppliers['providers'].setdefault(provider, []).append(applicationservice) + if hasattr(child, 'supplier'): + supplier = child.supplier + if ':' not in supplier: + providers_suppliers['suppliers'].setdefault(supplier, []).append(applicationservice) + else: + parse(applicationservice, elts + [child], dico, providers_suppliers, hidden) + + +applicationservices = listdir('seed') +#applicationservices = ['speedtest-rs'] +#applicationservices = ['redis'] +applicationservices_data = {} +tmps = {} +for applicationservice in applicationservices: + as_dir = join('seed', applicationservice) + if not isdir(as_dir): + continue + applicationservice_data = load_application_service(as_dir) + applicationservices_data[applicationservice] = {'description': applicationservice_data['description'], + 'website': applicationservice_data.get('website'), + 'as_dir': as_dir, + 'depends': [], + 'used_by': [], + } + if applicationservice in tmps: + applicationservices_data[applicationservice]['used_by'] = tmps.pop(applicationservice) + if 'depends' in applicationservice_data: + for depend in applicationservice_data['depends']: + applicationservices_data[applicationservice]['depends'].append(depend) + if depend in applicationservices_data: + applicationservices_data[depend]['used_by'].append(applicationservice) + else: + tmps.setdefault(depend, []).append(applicationservice) + +dico = {} +providers_suppliers = {'providers': {}, 'suppliers': {}} +for applicationservice, applicationservice_data in applicationservices_data.items(): + as_dir = applicationservice_data['as_dir'] + dirname = join(as_dir, 'dictionaries') + if isdir(dirname): + rougailconfig['dictionaries_dir'] = [dirname] + else: + rougailconfig['dictionaries_dir'] = [] + dirname_extras = join(as_dir, 'extras') + extra_dictionaries = {} + if isdir(dirname_extras): + for extra in listdir(dirname_extras): + extra_dir = join(dirname_extras, extra) + if isdir(extra_dir): + extra_dictionaries.setdefault(extra, []).append(extra_dir) + if not isdir(dirname) and not extra_dictionaries: + continue + rougailconfig['extra_dictionaries'] = extra_dictionaries + converted = RougailConvert(rougailconfig) + converted.load_dictionaries(just_doc=True) + converted.annotate() + objectspace = converted.rougailobjspace + if hasattr(objectspace.space, 'variables'): + dico[applicationservice] = {} + for name, elt in objectspace.space.variables.items(): + parse(applicationservice, [elt], dico[applicationservice], providers_suppliers, False) + + +def build_dependencies_tree(applicationservice, space): + depends = [] + if applicationservice_data['depends']: + for idx, depend in enumerate(applicationservices_data[applicationservice]['depends']): + subdepends = build_dependencies_tree(depend, space + 2) + if not idx or subdepends: + title = '\n' + else: + title = '' + title = ' ' * space + f'- [{depend}](../{depend}/README.md)' + depends.append(title) + depends.extend(subdepends) + return depends + + +for applicationservice, applicationservice_data in applicationservices_data.items(): + as_dir = applicationservice_data['as_dir'] + with open(join(as_dir, 'README.md'), 'w') as as_fh: + as_fh.write(f'---\ngitea: none\ninclude_toc: true\n---\n\n') + as_fh.write(f'# {applicationservice}\n\n') + as_fh.write(f'[All applications services for this dataset.](../README.md)\n\n') + as_fh.write(f'## Description\n\n') + description = applicationservice_data['description'] + '.\n' + if applicationservice_data['website']: + description += f'\n[For more informations]({applicationservice_data["website"]})\n' + as_fh.write(description) + if applicationservice_data['depends']: + as_fh.write(f'\n## Dependances\n\n') + for depend in build_dependencies_tree(applicationservice, 0): + as_fh.write(f'{depend}\n') + if applicationservice in dico and dico[applicationservice]: + as_fh.write('\n## Variables\n\n') + for title, data in dico[applicationservice].items(): + as_fh.write(f'{title}\n') + if data['type'] == 'leadership': + as_fh.write('\nThis a family is a leadership.\n') + if data['type'] == 'dynamic': + as_fh.write(f'\nThis a dynamic family generated from the variable "{data["suffixes"]}".\n') + if data['help']: + as_fh.write(f'\n{data["help"]}\n') + keys = [] + if data['variables']: + variables = data['variables'] + for variable in variables: + for key in variable: + if key not in keys: + keys.append(key) + values = [] + for variable in variables: + value = [] + for key in keys: + if key in variable: + val = variable[key] + elif key == 'type': + val = DEFAULT_TYPE + else: + val = '' + if key == 'type': + val = f'[{val}]({ROUGAIL_VARIABLE_TYPE})' + value.append(val) + values.append(value) + as_fh.write('\n') + as_fh.write(tabulate(values, headers=[key.capitalize() for key in keys], tablefmt="github")) + as_fh.write('\n') + as_fh.write('\n') + # FIXME if not applicationservice_data['used_by']: + # FIXME as_fh.write('\n## Variables with dependencies\n\n') + as_fh.write('\n- [+]: variable is multiple\n- **bold**: variable is mandatory\n') + if applicationservice_data['used_by']: + as_fh.write('\n## Used by\n\n') + for link in applicationservice_data['used_by']: + as_fh.write(f'- [{link}](../{link}/README.md)\n') + linked = [] + for provider, provider_as in providers_suppliers['providers'].items(): + if not applicationservice in provider_as: + continue + for supplier in providers_suppliers['suppliers'][provider]: + if not linked: + as_fh.write('\n## Linked to\n\n') + if supplier in linked: + continue + as_fh.write(f'- [{supplier}](../{supplier}/README.md)\n') + linked.append(supplier) + for supplier, supplier_as in providers_suppliers['suppliers'].items(): + if not applicationservice in supplier_as: + continue + for provider in providers_suppliers['providers'][supplier]: + if not linked: + as_fh.write('\n## Linked to\n\n') + if provider in linked: + continue + as_fh.write(f'- [{provider}](../{provider}/README.md)\n') + linked.append(provider) + + +with open('seed/README.md', 'w') as as_fh: + as_fh.write('# Application services\n\n') + applicationservices = {} + for applicationservice in applicationservices_data: + applicationservices.setdefault(applicationservice.split('-')[0], []).append(applicationservice) + applicationservice_categories = list(applicationservices.keys()) + applicationservice_categories.sort() + for category in applicationservice_categories: + applicationservices_ = applicationservices[category] + if len(applicationservices_) == 1: + applicationservice = applicationservices_[0] + applicationservice_data = applicationservices_data[applicationservice] + as_fh.write(f'- [{applicationservice}]({applicationservice}/README.md): {applicationservice_data["description"]}\n') + else: + as_fh.write(f'- {category}:\n') + applicationservices_.sort() + for applicationservice in applicationservices_: + applicationservice_data = applicationservices_data[applicationservice] + as_fh.write(f' - [{applicationservice}]({applicationservice}/README.md): {applicationservice_data["description"]}\n') diff --git a/src/risotto/image.py b/src/risotto/image.py index b80507e..63e4c46 100644 --- a/src/risotto/image.py +++ b/src/risotto/image.py @@ -24,6 +24,9 @@ class ModuleCfg(): def __repr__(self): return str(vars(self)) +def load_application_service(as_dir: str) -> str: + with open(join(as_dir, 'applicationservice.yml')) as yaml: + return yaml_load(yaml, Loader=SafeLoader) class Applications: def __init__(self) -> None: @@ -119,8 +122,7 @@ class Modules: self._load_applicationservice_directories(as_dir, cfg, ) - with open(join(as_dir, 'applicationservice.yml')) as yaml: - app = yaml_load(yaml, Loader=SafeLoader) + app = load_application_service(as_dir) provider = app.get('provider') if provider: cfg.providers.setdefault(provider, []) diff --git a/src/risotto/machine.py b/src/risotto/machine.py index ce3bef2..a04a255 100644 --- a/src/risotto/machine.py +++ b/src/risotto/machine.py @@ -1,4 +1,4 @@ -from .utils import MULTI_FUNCTIONS, load_zones, value_pprint, RISOTTO_CONFIG +from .utils import MULTI_FUNCTIONS, load_zones, value_pprint, RISOTTO_CONFIG, EXTRA_ANNOTATORS, ROUGAIL_NAMESPACE, ROUGAIL_NAMESPACE_DESCRIPTION from .image import Applications, Modules, valid_mandatories, applicationservice_copy from .rougail.annotator import calc_providers, calc_providers_global, calc_providers_dynamic, calc_providers_dynamic_follower, calc_providers_follower @@ -27,8 +27,6 @@ def tiramisu_display_name(kls, CONFIG_FILE = 'servers.yml' -ROUGAIL_NAMESPACE = 'general' -ROUGAIL_NAMESPACE_DESCRIPTION = 'Général' TIRAMISU_CACHE = 'tiramisu_cache.py' VALUES_CACHE = 'values_cache.json' INFORMATIONS_CACHE = 'informations_cache.json' @@ -148,7 +146,7 @@ class Loader: cfg['variable_namespace'] = ROUGAIL_NAMESPACE cfg['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION cfg['multi_functions'] = MULTI_FUNCTIONS - cfg['extra_annotators'] = ['risotto.rougail'] + cfg['extra_annotators'] = EXTRA_ANNOTATORS cfg['internal_functions'] = list(FUNCTIONS.keys()) cfg['force_convert_dyn_option_description'] = True cfg['risotto_globals'] = {} diff --git a/src/risotto/utils.py b/src/risotto/utils.py index f492093..c46a9ce 100644 --- a/src/risotto/utils.py +++ b/src/risotto/utils.py @@ -1,4 +1,5 @@ from os import environ +from os.path import isfile from typing import List from ipaddress import ip_address from toml import load as toml_load @@ -6,10 +7,17 @@ from pprint import pprint MULTI_FUNCTIONS = [] +EXTRA_ANNOTATORS = ['risotto.rougail'] +ROUGAIL_NAMESPACE = 'general' +ROUGAIL_NAMESPACE_DESCRIPTION = 'Général' -with open(environ.get('CONFIG_FILE', 'risotto.conf'), 'r') as fh: - RISOTTO_CONFIG = toml_load(fh) +config_file = environ.get('CONFIG_FILE', 'risotto.conf') +if isfile(config_file): + with open(config_file, 'r') as fh: + RISOTTO_CONFIG = toml_load(fh) +else: + RISOTTO_CONFIG = {} def _(s):