From 609914893bb90346c03f44663574bced7cb0c2db Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Fri, 25 Oct 2024 08:13:52 +0200 Subject: [PATCH] feat: can sort dictionaries in different directories --- src/rougail/__init__.py | 219 ++++++++++++++++++++++++++---- src/rougail/annotator/variable.py | 8 +- src/rougail/config.py | 35 ++++- src/rougail/convert.py | 116 +++++++++------- 4 files changed, 288 insertions(+), 90 deletions(-) diff --git a/src/rougail/__init__.py b/src/rougail/__init__.py index 80f9ede84..a383e9dad 100644 --- a/src/rougail/__init__.py +++ b/src/rougail/__init__.py @@ -27,10 +27,11 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """ -from tiramisu import Config -from tiramisu.error import PropertiesOptionError +from tiramisu import Config, undefined +from tiramisu.error import PropertiesOptionError, LeadershipError, ConfigError from warnings import warn from typing import List +from re import compile, findall from .convert import RougailConvert from .config import RougailConfig @@ -39,15 +40,20 @@ from .object_model import CONVERT_OPTION from .utils import normalize_family -def tiramisu_display_name(kls, subconfig) -> str: +def tiramisu_display_name(kls, + subconfig, + with_quote: bool=False, + ) -> str: """Replace the Tiramisu display_name function to display path + description""" doc = kls._get_information(subconfig, "doc", None) comment = f" ({doc})" if doc and doc != kls.impl_getname() else "" - if "{{ suffix }}" in comment: - comment = comment.replace('{{ suffix }}', str(subconfig.suffixes[-1])) + if "{{ identifier }}" in comment: + comment = comment.replace('{{ identifier }}', str(subconfig.identifiers[-1])) path = kls.impl_getpath() - if "{{ suffix }}" in path: - path = path.replace('{{ suffix }}', normalize_family(str(subconfig.suffixes[-1]))) + if "{{ identifier }}" in path and subconfig.identifiers: + path = path.replace('{{ identifier }}', normalize_family(str(subconfig.identifiers[-1]))) + if with_quote: + return f'"{path}"{comment}' return f"{path}{comment}" @@ -96,37 +102,194 @@ class Rougail: errors = [] warnings = [] for datas in user_datas: + options = datas.get('options', {}) for name, data in datas.get('values', {}).items(): - values.setdefault(name, {}).update(data) + values[name] = {'values': data, + 'options': options.copy(), + } errors.extend(datas.get('errors', [])) warnings.extend(datas.get('warnings', [])) + self._auto_configure_dynamics(values) while values: value_is_set = False - for option in self.config: - if option.path() in values and option.index() in values[option.path()]: - try: - option.value.set(values[option.path()]) - value_is_set = True - values.pop(option.path()) - except: - pass + for option in self._get_variable(self.config): + path = option.path() + if path not in values: + path = path.upper() + options = values.get(path, {}).get('options', {}) + if path not in values or options.get('upper') is not True: + continue + else: + options = values[path].get('options', {}) + value = values[path]["values"] + if option.ismulti(): + if options.get('multi_separator') and not isinstance(value, list): + value = value.split(options['multi_separator']) + values[path]["values"] = value + if options.get('needs_convert'): + value = [convert_value(option, val) for val in value] + values[path]["values"] = value + values[path]["options"]["needs_convert"] = False + elif options.get('needs_convert'): + value = convert_value(option, value) + index = option.index() + if index is not None: + if not isinstance(value, list) or index >= len(value): + continue + value = value[index] + try: + option.value.set(value) + value_is_set = True + if index is not None: + values[path]['values'][index] = undefined + if set(values[path]['values']) == {undefined}: + values.pop(path) + else: + values.pop(path) + except Exception as err: + if path != option.path(): + values[option.path()] = values.pop(path) if not value_is_set: break for path, data in values.items(): - for index, value in data.items(): - try: - print('attention', path, value) - self.config.option(path).value.set(value) - print('pfff') - except AttributeError as err: - errors.append(str(err)) - except ValueError as err: - errors.append(str(err).replace('"', "'")) - except PropertiesOptionError as err: - # warnings.append(f'"{err}" but is defined in "{self.filename}"') - warnings.append(str(err)) + try: + option = self.config.option(path) + value = data['values'] + if option.isfollower(): + for index, val in enumerate(value): + if val is undefined: + continue + self.config.option(path, index).value.set(val) + else: + option.value.set(value) + except AttributeError as err: + errors.append(str(err)) + except (ValueError, LeadershipError) as err: + #errors.append(str(err).replace('"', "'")) + errors.append(str(err)) + except PropertiesOptionError as err: +# warnings.append(f'"{err}" but is defined in "{self.filename}"') + warnings.append(str(err)) return {'errors': errors, 'warnings': warnings, } + def _get_variable(self, config): + for subconfig in config: + if subconfig.isoptiondescription(): + yield from self._get_variable(subconfig) + else: + yield subconfig + + def _auto_configure_dynamics(self, + values, + ): + cache = {} + added = [] + for path, data in list(values.items()): + value = data['values'] +# for value in data['values'].items(): + try: + option = self.config.option(path) + option.name() + except (ConfigError, PropertiesOptionError): + pass + except AttributeError: + config = self.config + current_path = '' + identifiers = [] + for name in path.split('.')[:-1]: + if current_path: + current_path += '.' + current_path += name + if current_path in cache: + config, identifier = cache[current_path] + identifiers.append(identifier) + else: + tconfig = config.option(name) + try: + tconfig.group_type() + config = tconfig + except AttributeError: + for tconfig in config.list(uncalculated=True): + if tconfig.isdynamic(only_self=True): + identifier = self._get_identifier(tconfig.name(), name) + if identifier is None: + continue + dynamic_variable = tconfig.information.get('dynamic_variable', + None, + ) + if not dynamic_variable: + continue + option_type = self.config.option(dynamic_variable).information.get('type') + if identifiers: + for s in identifiers: + dynamic_variable = dynamic_variable.replace('{{ identifier }}', str(s), 1) + if dynamic_variable not in values: + values[dynamic_variable] = {'values': []} + added.append(dynamic_variable) + elif dynamic_variable not in added: + continue + config = tconfig +# option_type = option.information.get('type') + typ = CONVERT_OPTION.get(option_type, {}).get("func") + if typ: + identifier = typ(identifier) + if identifier not in values[dynamic_variable]['values']: + values[dynamic_variable]['values'].append(identifier) + identifiers.append(identifier) + cache[current_path] = config, identifier + break + else: + if option.isdynamic(): + parent_option = self.config.option(path.rsplit('.', 1)[0]) + identifiers = self._get_identifier(parent_option.name(uncalculated=True), + parent_option.name(), + ) + dynamic_variable = None + while True: + dynamic_variable = parent_option.information.get('dynamic_variable', + None, + ) + if dynamic_variable: + break + parent_option = self.config.option(parent_option.path().rsplit('.', 1)[0]) + if '.' not in parent_option.path(): + parent_option = None + break + if not parent_option: + continue + identifiers = parent_option.identifiers() + for identifier in identifiers: + dynamic_variable = dynamic_variable.replace('{{ identifier }}', str(identifier), 1) + if dynamic_variable not in values: + values[dynamic_variable] = {'values': []} + added.append(dynamic_variable) + elif dynamic_variable not in added: + continue + option_type = option.information.get('type') + typ = CONVERT_OPTION.get(option_type, {}).get("func") + if typ: + identifier = typ(identifier) + if identifier not in values[dynamic_variable]['values']: + values[dynamic_variable]['values'].append(identifier) + cache[option.path()] = option, identifier + + def _get_identifier(self, true_name, name) -> str: + regexp = true_name.replace("{{ identifier }}", "(.*)") + finded = findall(regexp, name) + if len(finded) != 1 or not finded[0]: + return + return finded[0] + +def convert_value(option, value): + if value == '': + return None + option_type = option.information.get('type') + func = CONVERT_OPTION.get(option_type, {}).get("func") + if func: + return func(value) + return value + + __all__ = ("Rougail", "RougailConfig", "RougailUpgrade") diff --git a/src/rougail/annotator/variable.py b/src/rougail/annotator/variable.py index 8bedda6d6..7dfe41653 100644 --- a/src/rougail/annotator/variable.py +++ b/src/rougail/annotator/variable.py @@ -178,13 +178,7 @@ class Annotator(Walk): # pylint: disable=R0903 msg = _(f'the variable "{variable.path}" has regexp attribut but has not the "regexp" type') raise DictConsistencyError(msg, 37, variable.xmlfiles) if variable.mandatory is None: - family_path = variable.path - hidden = variable.hidden is True - while '.' in family_path and not hidden: - family_path = family_path.rsplit(".", 1)[0] - family = self.objectspace.paths[family_path] - hidden = family.hidden is True - variable.mandatory = not hidden + variable.mandatory = True def convert_test(self): """Convert variable tests value""" diff --git a/src/rougail/config.py b/src/rougail/config.py index 6698f41f5..5e1313f16 100644 --- a/src/rougail/config.py +++ b/src/rougail/config.py @@ -193,7 +193,7 @@ def get_rougail_config(*, main_namespace_default = 'rougail' else: main_namespace_default = 'null' - rougail_options = """default_dictionary_format_version: + rougail_options = f"""default_dictionary_format_version: description: Dictionary format version by default, if not specified in dictionary file alternative_name: v choices: @@ -219,7 +219,7 @@ sort_dictionaries_all: main_namespace: description: Main namespace name - default: MAIN_MAMESPACE_DEFAULT + default: {main_namespace_default} alternative_name: s mandatory: false @@ -289,18 +289,29 @@ functions_files: modes_level: description: All modes level available + multi: true + mandatory: false +""" + if backward_compatibility: + rougail_options += """ default: - basic - standard - advanced - commandline: false - +""" + rougail_options += """ default_family_mode: description: Default mode for a family default: - type: jinja jinja: | + {% if modes_level %} {{ modes_level[0] }} + {% endif %} + disabled: + jinja: | + {% if not modes_level %} + No mode + {% endif %} validators: - type: jinja jinja: | @@ -312,9 +323,19 @@ default_family_mode: default_variable_mode: description: Default mode for a variable default: - type: jinja jinja: | + {% if modes_level %} + {% if modes_level | length == 1 %} + {{ modes_level[0] }} + {% else %} {{ modes_level[1] }} + {% endif %} + {% endif %} + disabled: + jinja: | + {% if not modes_level %} + No mode + {% endif %} validators: - type: jinja jinja: | @@ -364,7 +385,7 @@ suffix: default: '' mandatory: false commandline: false -""".replace('MAIN_MAMESPACE_DEFAULT', main_namespace_default) +""" processes = {'structural': [], 'output': [], 'user data': [], diff --git a/src/rougail/convert.py b/src/rougail/convert.py index 21dcd94ff..2dc5e2900 100644 --- a/src/rougail/convert.py +++ b/src/rougail/convert.py @@ -133,7 +133,7 @@ class Paths: if not force and is_dynamic: self._dynamics[path] = dynamic - def get_relative_path(self, + def get_full_path(self, path: str, current_path: str, ): @@ -148,21 +148,22 @@ class Paths: def get_with_dynamic( self, path: str, - suffix_path: str, + identifier_path: str, current_path: str, version: str, namespace: str, xmlfiles: List[str], ) -> Any: - suffix = None + identifier = None if version != '1.0' and self.regexp_relative.search(path): - path = self.get_relative_path(path, - current_path, - ) + path = self.get_full_path(path, + current_path, + ) else: - path = get_realpath(path, suffix_path) + path = get_realpath(path, identifier_path) dynamic = None - if not path in self._data and "{{ suffix }}" not in path: + # version 1.0 + if version == "1.0" and not path in self._data and "{{ identifier }}" not in path: new_path = None current_path = None for name in path.split("."): @@ -184,10 +185,9 @@ class Paths: parent_dynamic = None name_dynamic = dynamic_path if ( - version == "1.0" - and parent_dynamic == parent_path - and name_dynamic.endswith("{{ suffix }}") - and name == name_dynamic.replace("{{ suffix }}", "") + parent_dynamic == parent_path + and name_dynamic.endswith("{{ identifier }}") + and name == name_dynamic.replace("{{ identifier }}", "") ): new_path += "." + name_dynamic break @@ -197,12 +197,12 @@ class Paths: else: new_path = name path = new_path - if not path in self._data: + if version != "1.0" and not path in self._data: current_path = None + parent_path = None new_path = current_path - suffixes = [] + identifiers = [] for name in path.split("."): - parent_path = current_path if current_path: current_path += "." + name else: @@ -213,6 +213,7 @@ class Paths: new_path += "." + name else: new_path = name + parent_path = current_path continue for dynamic_path in self._dynamics: if '.' in dynamic_path: @@ -221,30 +222,35 @@ class Paths: parent_dynamic = None name_dynamic = dynamic_path if ( - "{{ suffix }}" not in name_dynamic + "{{ identifier }}" not in name_dynamic or parent_path != parent_dynamic ): continue - regexp = "^" + name_dynamic.replace("{{ suffix }}", "(.*)") + regexp = "^" + name_dynamic.replace("{{ identifier }}", "(.*)") finded = findall(regexp, name) if len(finded) != 1 or not finded[0]: continue - suffixes.append(finded[0]) + if finded[0] == "{{ identifier }}": + identifiers.append(None) + else: + identifiers.append(finded[0]) if new_path is None: new_path = name_dynamic else: new_path += "." + name_dynamic + parent_path = dynamic_path break else: if new_path: new_path += "." + name else: new_path = name - if "{{ suffix }}" in name: - suffixes.append(None) + if "{{ identifier }}" in name: + identifiers.append(None) + parent_path = current_path path = new_path else: - suffixes = None + identifiers = None if path not in self._data: return None, None option = self._data[path] @@ -258,7 +264,7 @@ class Paths: f'shall not be used in the "{namespace}" namespace' ) raise DictConsistencyError(msg, 38, xmlfiles) - return option, suffixes + return option, identifiers def __getitem__( self, @@ -316,8 +322,8 @@ class Informations: class ParserVariable: def __init__(self, rougailconfig): - self.load_config(rougailconfig) self.rougailconfig = rougailconfig + self.load_config() self.paths = Paths(self.main_namespace) self.families = [] self.variables = [] @@ -342,12 +348,14 @@ class ParserVariable: self.is_init = False super().__init__() - def load_config(self, - rougailconfig: 'RougailConfig', - ) -> None: - self.rougailconfig = rougailconfig + def load_config(self) -> None: + rougailconfig = self.rougailconfig + self.sort_dictionaries_all = rougailconfig['sort_dictionaries_all'] + try: + self.main_dictionaries = rougailconfig["main_dictionaries"] + except: + self.main_dictionaries = [] self.main_namespace = rougailconfig["main_namespace"] - self.main_dictionaries = rougailconfig["main_dictionaries"] if self.main_namespace: self.extra_dictionaries = rougailconfig["extra_dictionaries"] self.suffix = rougailconfig["suffix"] @@ -355,14 +363,15 @@ class ParserVariable: self.custom_types = rougailconfig["custom_types"] self.functions_files = rougailconfig["functions_files"] self.modes_level = rougailconfig["modes_level"] - self.default_variable_mode = rougailconfig["default_variable_mode"] - self.default_family_mode = rougailconfig["default_family_mode"] + if self.modes_level: + self.default_variable_mode = rougailconfig["default_variable_mode"] + self.default_family_mode = rougailconfig["default_family_mode"] self.extra_annotators = rougailconfig["extra_annotators"] self.base_option_name = rougailconfig["base_option_name"] self.export_with_import = rougailconfig["export_with_import"] self.internal_functions = rougailconfig["internal_functions"] self.add_extra_options = rougailconfig["structural_commandline.add_extra_options"] - self.plugins = [] + self.plugins = rougailconfig["plugins"] def _init(self): if self.is_init: @@ -557,8 +566,10 @@ class ParserVariable: # it's just for modify subfamily or subvariable, do not redefine if family_obj: if not obj.pop("redefine", False): - raise Exception( - f"The family {path} already exists and she is not redefined in {filename}" + raise DictConsistencyError( + f'The family "{path}" already exists and it is not redefined', + 32, + [filename], ) # convert to Calculation objects self.parse_parameters( @@ -604,12 +615,12 @@ class ParserVariable: if obj_type == "dynamic": family_is_dynamic = True parent_dynamic = path - if '{{ suffix }}' not in name: + if '{{ identifier }}' not in name: if "variable" in family_obj: - name += '{{ suffix }}' - path += '{{ suffix }}' + name += '{{ identifier }}' + path += '{{ identifier }}' else: - msg = f'dynamic family name must have "{{{{ suffix }}}}" in his name for "{path}"' + msg = f'dynamic family name must have "{{{{ identifier }}}}" in his name for "{path}"' raise DictConsistencyError(msg, 13, [filename]) if version != '1.0' and not family_obj and comment: family_obj['description'] = comment @@ -938,6 +949,7 @@ class ParserVariable: variable["namespace"] = self.namespace variable["version"] = version + variable["path_prefix"] = self.path_prefix variable["xmlfiles"] = filename variable_type = self.get_family_or_variable_type(variable) obj = { @@ -1074,8 +1086,8 @@ class ParserVariable: try: params.append(PARAM_TYPES[param_typ](**val)) except ValidationError as err: - raise Exception( - f'"{attribute}" has an invalid "{key}" for {path}: {err}' + raise DictConsistencyError( + f'"{attribute}" has an invalid "{key}" for {path}: {err}', 29, xmlfiles ) from err calculation_object["params"] = params # @@ -1086,8 +1098,8 @@ class ParserVariable: f'unknown "return_type" in {attribute} of variable "{path}"' ) # - if typ == "suffix" and not family_is_dynamic: - msg = f'suffix calculation for "{attribute}" in "{path}" cannot be set none dynamic family' + if typ == "identifier" and not family_is_dynamic: + msg = f'identifier calculation for "{attribute}" in "{path}" cannot be set none dynamic family' raise DictConsistencyError(msg, 53, xmlfiles) if attribute in PROPERTY_ATTRIBUTE: calc = CALCULATION_PROPERTY_TYPES[typ](**calculation_object) @@ -1143,15 +1155,16 @@ class RougailConvert(ParserVariable): """Parse directories content""" self._init() if path_prefix: - if path_prefix in self.parents: + n_path_prefix = normalize_family(path_prefix) + if n_path_prefix in self.parents: raise Exception("pfffff") - root_parent = path_prefix - self.path_prefix = path_prefix + root_parent = n_path_prefix + self.path_prefix = n_path_prefix self.namespace = None self.add_family( - path_prefix, - path_prefix, - {}, + n_path_prefix, + n_path_prefix, + {'description': path_prefix}, "", False, None, @@ -1267,11 +1280,14 @@ class RougailConvert(ParserVariable): """Sort filename""" if not isinstance(directories, list): directories = [directories] + if self.sort_dictionaries_all: + filenames = {} for directory_name in directories: directory = Path(directory_name) if not directory.is_dir(): continue - filenames = {} + if not self.sort_dictionaries_all: + filenames = {} for file_path in directory.iterdir(): if file_path.suffix not in [".yml", ".yaml"]: continue @@ -1282,6 +1298,10 @@ class RougailConvert(ParserVariable): [filenames[file_path.name][1]], ) filenames[file_path.name] = str(file_path) + if not self.sort_dictionaries_all: + for filename in sorted(filenames): + yield filenames[filename] + if self.sort_dictionaries_all: for filename in sorted(filenames): yield filenames[filename]