From d2527f63532b0bdb12657bcae8cda6b22dd221bd Mon Sep 17 00:00:00 2001 From: gwen Date: Mon, 1 Jul 2024 18:04:18 +0200 Subject: [PATCH] default dictionary yaml format version --- .pre-commit-config.yaml | 47 +++++++ src/rougail/config.py | 4 +- src/rougail/convert.py | 280 +++++++++++++++++++++++----------------- src/rougail/update.py | 20 ++- 4 files changed, 220 insertions(+), 131 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..8deff9149 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,47 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-ast + - id: check-added-large-files + - id: check-json + - id: check-executables-have-shebangs + - id: check-symlinks + + - repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black + +# - repo: https://github.com/hhatto/autopep8 +# rev: v2.0.4 +# hooks: +# - id: autopep8 + +# - repo: https://github.com/pre-commit/mirrors-mypy +# rev: v1.6.1 +# hooks: +# - id: mypy + +# - repo: https://github.com/PyCQA/pylint +# rev: v3.0.2 +# hooks: +# - id: pylint + +# - repo: https://github.com/PyCQA/isort +# rev: 5.11.5 +# hooks: +# - id: isort + +# - repo: https://github.com/motet-a/jinjalint +# rev: 0.5 +# hooks: +# - id: jinjalint + +# - repo: https://github.com/rstcheck/rstcheck +# rev: v6.2.0 +# hooks: +# - id: rstcheck diff --git a/src/rougail/config.py b/src/rougail/config.py index 5bd0e2b3c..2e7f7bd5a 100644 --- a/src/rougail/config.py +++ b/src/rougail/config.py @@ -28,14 +28,14 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """ -from os.path import join, abspath, dirname - +from os.path import abspath, dirname, join ROUGAILROOT = "/srv/rougail" DTDDIR = join(dirname(abspath(__file__)), "data") RougailConfig = { + "default_dictionary_format_version": None, "dictionaries_dir": [join(ROUGAILROOT, "dictionaries")], "extra_dictionaries": {}, "services_dir": [join(ROUGAILROOT, "services")], diff --git a/src/rougail/convert.py b/src/rougail/convert.py index 6f04a7b2c..152bccb8c 100644 --- a/src/rougail/convert.py +++ b/src/rougail/convert.py @@ -29,46 +29,43 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """ import logging +from itertools import chain from pathlib import Path +from re import compile, findall from typing import ( - Optional, - Union, - get_type_hints, Any, - Literal, - List, Dict, Iterator, + List, + Literal, + Optional, Tuple, + Union, + get_type_hints, ) -from itertools import chain -from re import findall, compile +from pydantic import ValidationError from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap -from pydantic import ValidationError - from tiramisu.error import display_list -from .i18n import _ from .annotator import SpaceAnnotator -from .tiramisureflector import TiramisuReflector -from .utils import get_realpath +from .error import DictConsistencyError +from .i18n import _ +from .object_model import CONVERT_OPTION # Choice, from .object_model import ( - CONVERT_OPTION, - Family, - Dynamic, - Variable, - #Choice, - SymLink, CALCULATION_TYPES, - Calculation, - VariableCalculation, PARAM_TYPES, AnyParam, + Calculation, + Dynamic, + Family, + SymLink, + Variable, + VariableCalculation, ) -from .error import DictConsistencyError - +from .tiramisureflector import TiramisuReflector +from .utils import get_realpath property_types = Union[Literal[True], Calculation] properties_types = Dict[str, property_types] @@ -102,11 +99,12 @@ class Property: class Paths: _regexp_relative = compile(r"^_*\.(.*)$") - def __init__(self, - default_namespace: str, - ) -> None: + def __init__( + self, + default_namespace: str, + ) -> None: self._data: Dict[str, Union[Variable, Family]] = {} - self._dynamics: Dict[str: str] = {} + self._dynamics: Dict[str:str] = {} self.default_namespace = default_namespace self.path_prefix = None @@ -136,7 +134,7 @@ class Paths: xmlfiles: List[str], ) -> Any: suffix = None - if version != '1.0' and self._regexp_relative.search(path): + if version != "1.0" and self._regexp_relative.search(path): relative, subpath = path.split(".", 1) relative_len = len(relative) path_len = current_path.count(".") @@ -145,29 +143,34 @@ class Paths: else: path = get_realpath(path, suffix_path) dynamic = None - if not path in self._data and '{{ suffix }}' not in path: + if not path in self._data and "{{ suffix }}" not in path: new_path = None current_path = None - for name in path.split('.'): + for name in path.split("."): parent_path = current_path if current_path: - current_path += '.' + name + current_path += "." + name else: current_path = name if current_path in self._data: if new_path: - new_path += '.' + name + new_path += "." + name else: new_path = name continue for dynamic_path in self._dynamics: - parent_dynamic, name_dynamic = dynamic_path.rsplit('.', 1) - if version == '1.0' and parent_dynamic == parent_path and name_dynamic.endswith('{{ suffix }}') and name == name_dynamic.replace('{{ suffix }}', ''): - new_path += '.' + name_dynamic + parent_dynamic, name_dynamic = dynamic_path.rsplit(".", 1) + if ( + version == "1.0" + and parent_dynamic == parent_path + and name_dynamic.endswith("{{ suffix }}") + and name == name_dynamic.replace("{{ suffix }}", "") + ): + new_path += "." + name_dynamic break else: if new_path: - new_path += '.' + name + new_path += "." + name else: new_path = name path = new_path @@ -175,33 +178,36 @@ class Paths: current_path = None new_path = current_path suffixes = [] - for name in path.split('.'): + for name in path.split("."): parent_path = current_path if current_path: - current_path += '.' + name + current_path += "." + name else: current_path = name - #parent_path, name_path = path.rsplit('.', 1) + # parent_path, name_path = path.rsplit('.', 1) if current_path in self._data: if new_path: - new_path += '.' + name + new_path += "." + name else: new_path = name continue for dynamic_path in self._dynamics: - parent_dynamic, name_dynamic = dynamic_path.rsplit('.', 1) - if "{{ suffix }}" not in name_dynamic or parent_path != parent_dynamic: + parent_dynamic, name_dynamic = dynamic_path.rsplit(".", 1) + if ( + "{{ suffix }}" not in name_dynamic + or parent_path != parent_dynamic + ): continue regexp = "^" + name_dynamic.replace("{{ suffix }}", "(.*)") finded = findall(regexp, name) if len(finded) != 1 or not finded[0]: continue suffixes.append(finded[0]) - new_path += '.' + name_dynamic + new_path += "." + name_dynamic break else: if new_path: - new_path += '.' + name + new_path += "." + name else: new_path = name if "{{ suffix }}" in name: @@ -213,9 +219,14 @@ class Paths: return None, None option = self._data[path] option_namespace = option.namespace - if self.default_namespace not in [namespace, option_namespace] and namespace != option_namespace: - msg = _(f'A variable or a family located in the "{option_namespace}" namespace ' - f'shall not be used in the "{namespace}" namespace') + if ( + self.default_namespace not in [namespace, option_namespace] + and namespace != option_namespace + ): + msg = _( + f'A variable or a family located in the "{option_namespace}" namespace ' + f'shall not be used in the "{namespace}" namespace' + ) raise DictConsistencyError(msg, 38, xmlfiles) return option, suffixes @@ -292,7 +303,7 @@ class ParserVariable: # self.family = Family self.dynamic = Dynamic - self.choice = Variable #Choice + self.choice = Variable # Choice # self.exclude_imports = [] self.informations = Informations() @@ -309,7 +320,7 @@ class ParserVariable: self.variable = Variable hint = get_type_hints(self.dynamic) # FIXME: only for format 1.0 - hint['variable'] = str + hint["variable"] = str self.family_types = hint["type"].__args__ # pylint: disable=W0201 self.family_attrs = frozenset( # pylint: disable=W0201 set(hint) - {"name", "path", "xmlfiles"} | {"redefine"} @@ -320,7 +331,9 @@ class ParserVariable: # hint = get_type_hints(self.variable) - self.variable_types = self.convert_options #hint["type"].__args__ # pylint: disable=W0201 + self.variable_types = ( + self.convert_options + ) # hint["type"].__args__ # pylint: disable=W0201 # hint = get_type_hints(self.choice) self.choice_attrs = frozenset( # pylint: disable=W0201 @@ -379,7 +392,7 @@ class ParserVariable: else: return "variable" else: - if version == '1.0': + if version == "1.0": msg = f'Invalid value for the variable "{path}": "{obj}"' raise DictConsistencyError(msg, 102, [filename]) return "variable" @@ -420,7 +433,7 @@ class ParserVariable: msg = f'the variable or family name "{name}" is incorrect, it must not starts with "_" character' raise DictConsistencyError(msg, 16, [filename]) path = f"{subpath}.{name}" - if version == '0.1' and not isinstance(obj, dict) and obj is not None: + if version == "0.1" and not isinstance(obj, dict) and obj is not None: msg = f'the variable "{path}" has a wrong type "{type(obj)}"' raise DictConsistencyError(msg, 17, [filename]) typ = self.is_family_or_variable( @@ -460,7 +473,7 @@ class ParserVariable: first_variable: bool = False, family_is_leadership: bool = False, family_is_dynamic: bool = False, - parent_dynamic: Optional[str] = None + parent_dynamic: Optional[str] = None, ) -> None: """Parse a family""" if obj is None: @@ -505,14 +518,14 @@ class ParserVariable: if self.get_family_or_variable_type(family_obj) == "dynamic": family_is_dynamic = True parent_dynamic = path - if version == '1.0' and '{{ suffix }}' not in name: - name += '{{ suffix }}' - path += '{{ suffix }}' - if '{{ suffix }}' not in name: + if version == "1.0" and "{{ suffix }}" not in name: + name += "{{ suffix }}" + path += "{{ suffix }}" + if "{{ suffix }}" not in name: msg = f'dynamic family name must have "{{{{ suffix }}}}" in his name for "{path}"' raise DictConsistencyError(msg, 13, [filename]) - if version != '1.0' and not family_obj and comment: - family_obj['description'] = comment + if version != "1.0" and not family_obj and comment: + family_obj["description"] = comment self.add_family( path, name, @@ -592,29 +605,41 @@ class ParserVariable: family_obj = self.dynamic if version == "1.0": if "variable" not in family: - raise DictConsistencyError(f'dynamic family must have "variable" attribute for "{path}"', 101, family["xmlfiles"]) - if 'dynamic' in family: - raise DictConsistencyError('variable and dynamic cannot be set together in the dynamic family "{path}"', 100, family['xmlfiles']) - family['dynamic'] = {'type': 'variable', - 'variable': family['variable'], - 'propertyerror': False, - 'allow_none': True, - } - del family['variable'] - #FIXME only for 1.0 + raise DictConsistencyError( + f'dynamic family must have "variable" attribute for "{path}"', + 101, + family["xmlfiles"], + ) + if "dynamic" in family: + raise DictConsistencyError( + 'variable and dynamic cannot be set together in the dynamic family "{path}"', + 100, + family["xmlfiles"], + ) + family["dynamic"] = { + "type": "variable", + "variable": family["variable"], + "propertyerror": False, + "allow_none": True, + } + del family["variable"] + # FIXME only for 1.0 if "variable" in family: - raise Exception(f'dynamic family must not have "variable" attribute for "{family["path"]}" in {family["xmlfiles"]}') + raise Exception( + f'dynamic family must not have "variable" attribute for "{family["path"]}" in {family["xmlfiles"]}' + ) else: family_obj = self.family # convert to Calculation objects - self.parse_parameters(path, - family, - filename, - family_is_dynamic, - False, - version, - typ='family', - ) + self.parse_parameters( + path, + family, + filename, + family_is_dynamic, + False, + version, + typ="family", + ) try: self.paths.add( path, @@ -651,27 +676,28 @@ class ParserVariable: parent_dynamic: Optional[str] = None, ) -> None: """Parse variable""" - if version == '1.0' or isinstance(obj, dict): + if version == "1.0" or isinstance(obj, dict): if obj is None: obj = {} extra_attrs = set(obj) - self.choice_attrs else: extra_attrs = [] - obj = {'default': obj} + obj = {"default": obj} if comment: - obj['description'] = comment + obj["description"] = comment if extra_attrs: raise Exception( f'"{path}" is not a valid variable, there are additional ' f'attributes: "{", ".join(extra_attrs)}"' ) - self.parse_parameters(path, - obj, - filename, - family_is_dynamic, - family_is_leadership is True and first_variable is False, - version, - ) + self.parse_parameters( + path, + obj, + filename, + family_is_dynamic, + family_is_leadership is True and first_variable is False, + version, + ) self.parse_params(path, obj) if path in self.paths: if "exists" in obj and not obj.pop("exists"): @@ -680,7 +706,11 @@ class ParserVariable: msg = f'Variable "{path}" already exists' raise DictConsistencyError(msg, 45, [filename]) self.paths.add( - path, self.paths[path].model_copy(update=obj),family_is_dynamic, parent_dynamic, force=True + path, + self.paths[path].model_copy(update=obj), + family_is_dynamic, + parent_dynamic, + force=True, ) self.paths[path].xmlfiles.append(filename) else: @@ -694,12 +724,7 @@ class ParserVariable: raise DictConsistencyError(msg, 46, [filename]) obj["path"] = path self.add_variable( - name, - obj, - filename, - family_is_dynamic, - parent_dynamic, - version + name, obj, filename, family_is_dynamic, parent_dynamic, version ) if family_is_leadership: if first_variable: @@ -707,18 +732,19 @@ class ParserVariable: else: self.followers.append(path) - def parse_parameters(self, - path: str, - obj: dict, - filename: str, - family_is_dynamic: bool, - is_follower: bool, - version: str, - *, - typ: str='variable', - ): + def parse_parameters( + self, + path: str, + obj: dict, + filename: str, + family_is_dynamic: bool, + is_follower: bool, + version: str, + *, + typ: str = "variable", + ): """Parse variable or family parameters""" - if typ == 'variable': + if typ == "variable": calculations = self.choice_calculations else: calculations = self.family_calculations @@ -783,7 +809,18 @@ class ParserVariable: params = [] for key, val in obj["params"].items(): try: - params.append(AnyParam(key=key, value=val, type="any", path=None, is_follower=None, attribute=None, family_is_dynamic=None, xmlfiles=None)) + params.append( + AnyParam( + key=key, + value=val, + type="any", + path=None, + is_follower=None, + attribute=None, + family_is_dynamic=None, + xmlfiles=None, + ) + ) except ValidationError as err: raise Exception( f'"{key}" has an invalid "params" for {path}: {err}' @@ -797,7 +834,7 @@ class ParserVariable: filename: str, family_is_dynamic: bool, parent_dynamic: Optional[str], - version: str + version: str, ) -> None: """Add a new variable""" if not isinstance(filename, list): @@ -916,11 +953,11 @@ class ParserVariable: else: param_typ = val["type"] val["key"] = key - val['path'] = path - val['family_is_dynamic'] = family_is_dynamic - val['is_follower'] = is_follower - val['attribute'] = attribute - val['xmlfiles'] = xmlfiles + val["path"] = path + val["family_is_dynamic"] = family_is_dynamic + val["is_follower"] = is_follower + val["attribute"] = attribute + val["xmlfiles"] = xmlfiles try: params.append(PARAM_TYPES[param_typ](**val)) except ValidationError as err: @@ -1031,10 +1068,11 @@ class RougailConvert(ParserVariable): if path_prefix: self.path_prefix = None - def get_comment(self, - name: str, - objects: CommentedMap, - ) -> Optional[str]: + def get_comment( + self, + name: str, + objects: CommentedMap, + ) -> Optional[str]: if name in objects.ca.items: comment = objects.ca.items[name][2] else: @@ -1110,8 +1148,14 @@ class RougailConvert(ParserVariable): version = str(obj.pop(name)) break else: - msg = '"version" attribut is mandatory in YAML file' - raise DictConsistencyError(msg, 27, [filename]) + # the `version` attribute is not mandatory + default_version = self.rougailconfig["default_dictionary_format_version"] + if default_version is not None: + version = default_version + else: + msg = '"version" attribut is mandatory in YAML file' + raise DictConsistencyError(msg, 27, [filename]) + if version not in self.supported_version: msg = f'version "{version}" is not supported, list of supported versions: {display_list(self.supported_version, separator="or", add_quote=True)}' raise DictConsistencyError(msg, 28, [filename]) diff --git a/src/rougail/update.py b/src/rougail/update.py index bd30ea881..44ca4eb35 100644 --- a/src/rougail/update.py +++ b/src/rougail/update.py @@ -23,29 +23,27 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """ -from typing import List, Any, Optional, Tuple -from os.path import join, isfile, isdir, basename from os import listdir, makedirs +from os.path import basename, isdir, isfile, join +from typing import Any, List, Optional, Tuple try: - from lxml.etree import parse, XMLParser, XMLSyntaxError # pylint: disable=E0611 - from lxml.etree import Element, SubElement, tostring + from lxml.etree import SubElement # pylint: disable=E0611 + from lxml.etree import Element, XMLParser, XMLSyntaxError, parse, tostring except ModuleNotFoundError as err: parse = None # from ast import parse as ast_parse from json import dumps -from ruamel.yaml import YAML -from yaml import dump, SafeDumper from pathlib import Path -from .i18n import _ -from .error import UpgradeError +from ruamel.yaml import YAML -from .utils import normalize_family from .config import RougailConfig +from .error import UpgradeError +from .i18n import _ from .object_model import CONVERT_OPTION - +from .utils import normalize_family VERSIONS = ["0.10", "1.0", "1.1"] @@ -644,7 +642,7 @@ class RougailUpgrade: ext = "xml" else: with xmlsrc.open() as xml_fh: - root = YAML(typ='safe').load(file_fh) + root = YAML(typ="safe").load(file_fh) search_function_name = get_function_name(str(root["version"])) ext = "yml" function_found = False