Merge branch 'float_version' into develop

.pre-commit-config.yaml

@@ -0,0 +1,47 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-ast
+      - id: check-added-large-files
+      - id: check-json
+      - id: check-executables-have-shebangs
+      - id: check-symlinks
+
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+      - id: black
+
+#  - repo: https://github.com/hhatto/autopep8
+#    rev: v2.0.4
+#    hooks:
+#      - id: autopep8
+
+#  - repo: https://github.com/pre-commit/mirrors-mypy
+#    rev: v1.6.1
+#    hooks:
+#      - id: mypy
+
+#  - repo: https://github.com/PyCQA/pylint
+#    rev: v3.0.2
+#    hooks:
+#      - id: pylint
+
+#  - repo: https://github.com/PyCQA/isort
+#    rev: 5.11.5
+#    hooks:
+#      - id: isort
+
+#  - repo: https://github.com/motet-a/jinjalint
+#    rev: 0.5
+#    hooks:
+#      - id: jinjalint
+
+# - repo: https://github.com/rstcheck/rstcheck
+#   rev: v6.2.0
+#   hooks:
+#     - id: rstcheck

docs/developer.rst

@@ -14,3 +14,47 @@ This process describes how to install and run the project locally, e.g. for deve
 *Nota*: command is to be executed through the terminal
 
 `pip install rougail`
+
+Code quality
+---------------
+
+We are using `pre-commit <https://pre-commit.com/>`_, there is a :file:`.pre-commit-config.yaml`
+pre-commit config file in the root's project.
+
+You need to:
+
+- install the pre-commit library::
+
+    pip install pre-commit
+
+- registrer the pre-commit git hooks with this command::
+
+    pre-commit install
+
+- launch the quality code procedure with::
+
+    pre-commit
+
+  or simply just commit your changes, pre-commit will automatically be launched.
+
+.. attention:: If an error is found, the commit will not happen.
+   You must resolve all errors that pre-commit that pre-commit points out to you before.
+
+.. note:: If you need for some reason to disable `pre-commit`, just set
+          the `PRE_COMMIT_ALLOW_NO_CONFIG` environment variable before commiting::
+
+              PRE_COMMIT_ALLOW_NO_CONFIG=1 git commit
+
+Coding standard
+------------------
+
+We use black
+
+.. code-block:: yaml
+
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+      - id: black
+
+And some YAML and JSON validators.

pyproject.toml

@@ -26,11 +26,14 @@ dependencies = [
     "ruamel.yaml ~= 0.17.40",
     "pydantic ~= 2.5.2",
     "jinja2 ~= 3.1.2",
-    "tiramisu ~= 4.1.0",
+    "tiramisu ~= 4.1.0"
 ]
-[project.optional-dependancies]
+
+[project.optional-dependencies]
 dev = [
     "pylint ~= 3.0.3",
+    "pytest ~= 8.2.2",
+    "lxml ~= 5.2.2"
 ]
 
 [tool.commitizen]

src/rougail/config.py

@@ -28,14 +28,14 @@ You should have received a copy of the GNU General Public License
 along with this program; if not, write to the Free Software
 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 """
-from os.path import join, abspath, dirname
+from os.path import abspath, dirname, join
-
 
 ROUGAILROOT = "/srv/rougail"
 DTDDIR = join(dirname(abspath(__file__)), "data")
 
 
 RougailConfig = {
+    "default_dictionary_format_version": None,
     "dictionaries_dir": [join(ROUGAILROOT, "dictionaries")],
     "extra_dictionaries": {},
     "services_dir": [join(ROUGAILROOT, "services")],

src/rougail/convert.py

@@ -29,46 +29,43 @@ along with this program; if not, write to the Free Software
 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 """
 import logging
+from itertools import chain
 from pathlib import Path
+from re import compile, findall
 from typing import (
-    Optional,
-    Union,
-    get_type_hints,
     Any,
-    Literal,
-    List,
     Dict,
     Iterator,
+    List,
+    Literal,
+    Optional,
     Tuple,
+    Union,
+    get_type_hints,
 )
-from itertools import chain
-from re import findall, compile
 
+from pydantic import ValidationError
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
-from pydantic import ValidationError
-
 from tiramisu.error import display_list
 
-from .i18n import _
 from .annotator import SpaceAnnotator
-from .tiramisureflector import TiramisuReflector
+from .error import DictConsistencyError
-from .utils import get_realpath
+from .i18n import _
+from .object_model import CONVERT_OPTION  # Choice,
 from .object_model import (
-    CONVERT_OPTION,
-    Family,
-    Dynamic,
-    Variable,
-    #Choice,
-    SymLink,
     CALCULATION_TYPES,
-    Calculation,
-    VariableCalculation,
     PARAM_TYPES,
     AnyParam,
+    Calculation,
+    Dynamic,
+    Family,
+    SymLink,
+    Variable,
+    VariableCalculation,
 )
-from .error import DictConsistencyError
+from .tiramisureflector import TiramisuReflector
-
+from .utils import get_realpath
 
 property_types = Union[Literal[True], Calculation]
 properties_types = Dict[str, property_types]
@@ -102,7 +99,8 @@ class Property:
 class Paths:
     _regexp_relative = compile(r"^_*\.(.*)$")
 
-    def __init__(self,
+    def __init__(
+        self,
         default_namespace: str,
     ) -> None:
         self._data: Dict[str, Union[Variable, Family]] = {}
@@ -136,7 +134,7 @@ class Paths:
         xmlfiles: List[str],
     ) -> Any:
         suffix = None
-        if version != '1.0' and self._regexp_relative.search(path):
+        if version != "1.0" and self._regexp_relative.search(path):
             relative, subpath = path.split(".", 1)
             relative_len = len(relative)
             path_len = current_path.count(".")
@@ -145,29 +143,34 @@ class Paths:
         else:
             path = get_realpath(path, suffix_path)
         dynamic = None
-        if not path in self._data and '{{ suffix }}' not in path:
+        if not path in self._data and "{{ suffix }}" not in path:
             new_path = None
             current_path = None
-            for name in path.split('.'):
+            for name in path.split("."):
                 parent_path = current_path
                 if current_path:
-                    current_path += '.' + name
+                    current_path += "." + name
                 else:
                     current_path = name
                 if current_path in self._data:
                     if new_path:
-                        new_path += '.' + name
+                        new_path += "." + name
                     else:
                         new_path = name
                     continue
                 for dynamic_path in self._dynamics:
-                    parent_dynamic, name_dynamic = dynamic_path.rsplit('.', 1)
+                    parent_dynamic, name_dynamic = dynamic_path.rsplit(".", 1)
-                    if version == '1.0' and parent_dynamic == parent_path and name_dynamic.endswith('{{ suffix }}') and name == name_dynamic.replace('{{ suffix }}', ''):
+                    if (
-                        new_path += '.' + name_dynamic
+                        version == "1.0"
+                        and parent_dynamic == parent_path
+                        and name_dynamic.endswith("{{ suffix }}")
+                        and name == name_dynamic.replace("{{ suffix }}", "")
+                    ):
+                        new_path += "." + name_dynamic
                         break
                 else:
                     if new_path:
-                        new_path += '.' + name
+                        new_path += "." + name
                     else:
                         new_path = name
             path = new_path
@@ -175,33 +178,36 @@ class Paths:
             current_path = None
             new_path = current_path
             suffixes = []
-            for name in path.split('.'):
+            for name in path.split("."):
                 parent_path = current_path
                 if current_path:
-                    current_path += '.' + name
+                    current_path += "." + name
                 else:
                     current_path = name
                 # parent_path, name_path = path.rsplit('.', 1)
                 if current_path in self._data:
                     if new_path:
-                        new_path += '.' + name
+                        new_path += "." + name
                     else:
                         new_path = name
                     continue
                 for dynamic_path in self._dynamics:
-                    parent_dynamic, name_dynamic = dynamic_path.rsplit('.', 1)
+                    parent_dynamic, name_dynamic = dynamic_path.rsplit(".", 1)
-                    if "{{ suffix }}" not in name_dynamic or parent_path != parent_dynamic:
+                    if (
+                        "{{ suffix }}" not in name_dynamic
+                        or parent_path != parent_dynamic
+                    ):
                         continue
                     regexp = "^" + name_dynamic.replace("{{ suffix }}", "(.*)")
                     finded = findall(regexp, name)
                     if len(finded) != 1 or not finded[0]:
                         continue
                     suffixes.append(finded[0])
-                    new_path += '.' + name_dynamic
+                    new_path += "." + name_dynamic
                     break
                 else:
                     if new_path:
-                        new_path += '.' + name
+                        new_path += "." + name
                     else:
                         new_path = name
                     if "{{ suffix }}" in name:
@@ -213,9 +219,14 @@ class Paths:
             return None, None
         option = self._data[path]
         option_namespace = option.namespace
-        if self.default_namespace not in [namespace, option_namespace] and namespace != option_namespace:
+        if (
-            msg = _(f'A variable or a family located in the "{option_namespace}" namespace '
+            self.default_namespace not in [namespace, option_namespace]
-            f'shall not be used in the "{namespace}" namespace')
+            and namespace != option_namespace
+        ):
+            msg = _(
+                f'A variable or a family located in the "{option_namespace}" namespace '
+                f'shall not be used in the "{namespace}" namespace'
+            )
             raise DictConsistencyError(msg, 38, xmlfiles)
         return option, suffixes
 
@@ -309,7 +320,7 @@ class ParserVariable:
         self.variable = Variable
         hint = get_type_hints(self.dynamic)
         # FIXME: only for format 1.0
-        hint['variable'] = str
+        hint["variable"] = str
         self.family_types = hint["type"].__args__  # pylint: disable=W0201
         self.family_attrs = frozenset(  # pylint: disable=W0201
             set(hint) - {"name", "path", "xmlfiles"} | {"redefine"}
@@ -320,7 +331,9 @@ class ParserVariable:
         #
         hint = get_type_hints(self.variable)
 
-        self.variable_types = self.convert_options #hint["type"].__args__  # pylint: disable=W0201
+        self.variable_types = (
+            self.convert_options
+        )  # hint["type"].__args__  # pylint: disable=W0201
         #
         hint = get_type_hints(self.choice)
         self.choice_attrs = frozenset(  # pylint: disable=W0201
@@ -379,7 +392,7 @@ class ParserVariable:
                 else:
                     return "variable"
         else:
-            if version == '1.0':
+            if version == "1.0":
                 msg = f'Invalid value for the variable "{path}": "{obj}"'
                 raise DictConsistencyError(msg, 102, [filename])
             return "variable"
@@ -420,7 +433,7 @@ class ParserVariable:
             msg = f'the variable or family name "{name}" is incorrect, it must not starts with "_" character'
             raise DictConsistencyError(msg, 16, [filename])
         path = f"{subpath}.{name}"
-        if version == '0.1' and not isinstance(obj, dict) and obj is not None:
+        if version == "0.1" and not isinstance(obj, dict) and obj is not None:
             msg = f'the variable "{path}" has a wrong type "{type(obj)}"'
             raise DictConsistencyError(msg, 17, [filename])
         typ = self.is_family_or_variable(
@@ -460,7 +473,7 @@ class ParserVariable:
         first_variable: bool = False,
         family_is_leadership: bool = False,
         family_is_dynamic: bool = False,
-        parent_dynamic: Optional[str] = None
+        parent_dynamic: Optional[str] = None,
     ) -> None:
         """Parse a family"""
         if obj is None:
@@ -505,14 +518,14 @@ class ParserVariable:
             if self.get_family_or_variable_type(family_obj) == "dynamic":
                 family_is_dynamic = True
                 parent_dynamic = path
-                if version == '1.0' and '{{ suffix }}' not in name:
+                if version == "1.0" and "{{ suffix }}" not in name:
-                    name += '{{ suffix }}'
+                    name += "{{ suffix }}"
-                    path += '{{ suffix }}'
+                    path += "{{ suffix }}"
-                if '{{ suffix }}' not in name:
+                if "{{ suffix }}" not in name:
                     msg = f'dynamic family name must have "{{{{ suffix }}}}" in his name for "{path}"'
                     raise DictConsistencyError(msg, 13, [filename])
-            if version != '1.0' and not family_obj and comment:
+            if version != "1.0" and not family_obj and comment:
-                family_obj['description'] = comment
+                family_obj["description"] = comment
             self.add_family(
                 path,
                 name,
@@ -592,28 +605,40 @@ class ParserVariable:
             family_obj = self.dynamic
             if version == "1.0":
                 if "variable" not in family:
-                    raise DictConsistencyError(f'dynamic family must have "variable" attribute for "{path}"', 101, family["xmlfiles"])
+                    raise DictConsistencyError(
-                if 'dynamic' in family:
+                        f'dynamic family must have "variable" attribute for "{path}"',
-                    raise DictConsistencyError('variable and dynamic cannot be set together in the dynamic family "{path}"', 100, family['xmlfiles'])
+                        101,
-                family['dynamic'] = {'type': 'variable',
+                        family["xmlfiles"],
-                                     'variable': family['variable'],
+                    )
-                                     'propertyerror': False,
+                if "dynamic" in family:
-                                     'allow_none': True,
+                    raise DictConsistencyError(
+                        'variable and dynamic cannot be set together in the dynamic family "{path}"',
+                        100,
+                        family["xmlfiles"],
+                    )
+                family["dynamic"] = {
+                    "type": "variable",
+                    "variable": family["variable"],
+                    "propertyerror": False,
+                    "allow_none": True,
                 }
-                del family['variable']
+                del family["variable"]
             # FIXME only for 1.0
             if "variable" in family:
-                raise Exception(f'dynamic family must not have "variable" attribute for "{family["path"]}" in {family["xmlfiles"]}')
+                raise Exception(
+                    f'dynamic family must not have "variable" attribute for "{family["path"]}" in {family["xmlfiles"]}'
+                )
         else:
             family_obj = self.family
         # convert to Calculation objects
-        self.parse_parameters(path,
+        self.parse_parameters(
+            path,
             family,
             filename,
             family_is_dynamic,
             False,
             version,
-                              typ='family',
+            typ="family",
         )
         try:
             self.paths.add(
@@ -651,21 +676,22 @@ class ParserVariable:
         parent_dynamic: Optional[str] = None,
     ) -> None:
         """Parse variable"""
-        if version == '1.0' or isinstance(obj, dict):
+        if version == "1.0" or isinstance(obj, dict):
             if obj is None:
                 obj = {}
             extra_attrs = set(obj) - self.choice_attrs
         else:
             extra_attrs = []
-            obj = {'default': obj}
+            obj = {"default": obj}
             if comment:
-                obj['description'] = comment
+                obj["description"] = comment
         if extra_attrs:
             raise Exception(
                 f'"{path}" is not a valid variable, there are additional '
                 f'attributes: "{", ".join(extra_attrs)}"'
             )
-        self.parse_parameters(path,
+        self.parse_parameters(
+            path,
             obj,
             filename,
             family_is_dynamic,
@@ -680,7 +706,11 @@ class ParserVariable:
                 msg = f'Variable "{path}" already exists'
                 raise DictConsistencyError(msg, 45, [filename])
             self.paths.add(
-                path, self.paths[path].model_copy(update=obj),family_is_dynamic, parent_dynamic, force=True
+                path,
+                self.paths[path].model_copy(update=obj),
+                family_is_dynamic,
+                parent_dynamic,
+                force=True,
             )
             self.paths[path].xmlfiles.append(filename)
         else:
@@ -694,12 +724,7 @@ class ParserVariable:
                 raise DictConsistencyError(msg, 46, [filename])
             obj["path"] = path
             self.add_variable(
-                name,
+                name, obj, filename, family_is_dynamic, parent_dynamic, version
-                obj,
-                filename,
-                family_is_dynamic,
-                parent_dynamic,
-                version
             )
             if family_is_leadership:
                 if first_variable:
@@ -707,7 +732,8 @@ class ParserVariable:
                 else:
                     self.followers.append(path)
 
-    def parse_parameters(self,
+    def parse_parameters(
+        self,
         path: str,
         obj: dict,
         filename: str,
@@ -715,10 +741,10 @@ class ParserVariable:
         is_follower: bool,
         version: str,
         *,
-                         typ: str='variable',
+        typ: str = "variable",
     ):
         """Parse variable or family parameters"""
-        if typ == 'variable':
+        if typ == "variable":
             calculations = self.choice_calculations
         else:
             calculations = self.family_calculations
@@ -783,7 +809,18 @@ class ParserVariable:
         params = []
         for key, val in obj["params"].items():
             try:
-                params.append(AnyParam(key=key, value=val, type="any", path=None, is_follower=None, attribute=None, family_is_dynamic=None, xmlfiles=None))
+                params.append(
+                    AnyParam(
+                        key=key,
+                        value=val,
+                        type="any",
+                        path=None,
+                        is_follower=None,
+                        attribute=None,
+                        family_is_dynamic=None,
+                        xmlfiles=None,
+                    )
+                )
             except ValidationError as err:
                 raise Exception(
                     f'"{key}" has an invalid "params" for {path}: {err}'
@@ -797,7 +834,7 @@ class ParserVariable:
         filename: str,
         family_is_dynamic: bool,
         parent_dynamic: Optional[str],
-        version: str
+        version: str,
     ) -> None:
         """Add a new variable"""
         if not isinstance(filename, list):
@@ -916,11 +953,11 @@ class ParserVariable:
                 else:
                     param_typ = val["type"]
                 val["key"] = key
-                val['path'] = path
+                val["path"] = path
-                val['family_is_dynamic'] = family_is_dynamic
+                val["family_is_dynamic"] = family_is_dynamic
-                val['is_follower'] = is_follower
+                val["is_follower"] = is_follower
-                val['attribute'] = attribute
+                val["attribute"] = attribute
-                val['xmlfiles'] = xmlfiles
+                val["xmlfiles"] = xmlfiles
                 try:
                     params.append(PARAM_TYPES[param_typ](**val))
                 except ValidationError as err:
@@ -1031,7 +1068,8 @@ class RougailConvert(ParserVariable):
         if path_prefix:
             self.path_prefix = None
 
-    def get_comment(self,
+    def get_comment(
+        self,
         name: str,
         objects: CommentedMap,
     ) -> Optional[str]:
@@ -1109,9 +1147,15 @@ class RougailConvert(ParserVariable):
                 continue
             version = str(obj.pop(name))
             break
+        else:
+            # the `version` attribute is not mandatory
+            default_version = self.rougailconfig["default_dictionary_format_version"]
+            if default_version is not None:
+                version = default_version
             else:
                 msg = '"version" attribut is mandatory in YAML file'
                 raise DictConsistencyError(msg, 27, [filename])
+
         if version not in self.supported_version:
             msg = f'version "{version}" is not supported, list of supported versions: {display_list(self.supported_version, separator="or", add_quote=True)}'
             raise DictConsistencyError(msg, 28, [filename])

src/rougail/update.py

@@ -23,29 +23,27 @@ along with this program; if not, write to the Free Software
 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 """
 
-from typing import List, Any, Optional, Tuple
-from os.path import join, isfile, isdir, basename
 from os import listdir, makedirs
+from os.path import basename, isdir, isfile, join
+from typing import Any, List, Optional, Tuple
 
 try:
-    from lxml.etree import parse, XMLParser, XMLSyntaxError  # pylint: disable=E0611
+    from lxml.etree import SubElement  # pylint: disable=E0611
-    from lxml.etree import Element, SubElement, tostring
+    from lxml.etree import Element, XMLParser, XMLSyntaxError, parse, tostring
 except ModuleNotFoundError as err:
     parse = None
 
 # from ast import parse as ast_parse
 from json import dumps
-from ruamel.yaml import YAML
-from yaml import dump, SafeDumper
 from pathlib import Path
 
-from .i18n import _
+from ruamel.yaml import YAML
-from .error import UpgradeError
 
-from .utils import normalize_family
 from .config import RougailConfig
+from .error import UpgradeError
+from .i18n import _
 from .object_model import CONVERT_OPTION
-
+from .utils import normalize_family
 
 VERSIONS = ["0.10", "1.0", "1.1"]
 
@@ -644,7 +642,7 @@ class RougailUpgrade:
                 ext = "xml"
             else:
                 with xmlsrc.open() as xml_fh:
-                    root = YAML(typ='safe').load(file_fh)
+                    root = YAML(typ="safe").load(file_fh)
                 search_function_name = get_function_name(str(root["version"]))
                 ext = "yml"
             function_found = False

tests/data/dict1/dict.yml

@@ -0,0 +1,5 @@
+# dict with a correct version declared
+#version: "1.1"
+hello:
+  type: string
+  default: world

tests/data/dict2/dict.yml

@@ -0,0 +1,5 @@
+# dict with a correct version declared
+version: "1.0"
+hello:
+  type: string
+  default: world

tests/test_version.py

@@ -0,0 +1,82 @@
+from shutil import rmtree  # , copyfile, copytree
+from os import getcwd, makedirs
+from os.path import isfile, join, isdir
+from pytest import fixture, raises
+from os import listdir
+
+from rougail import Rougail, RougailConfig
+from rougail.error import DictConsistencyError
+
+from ruamel.yaml import YAML
+
+# dico_dirs = 'tests/data'
+# test_ok = set()
+# for test in listdir(dico_dirs):
+#    if isdir(join(dico_dirs, test)):
+#        test_ok.add(test)
+# excludes = set([])
+# test_ok -= excludes
+
+# ORI_DIR = getcwd()
+
+# test_ok = list(test_ok)
+# test_ok.sort()
+##print(test_ok)
+
+# @fixture(scope="module", params=test_ok)
+# def test_dir(request):
+#    return request.param
+
+"""
+the kinematics are as follows:
+- if no version attribute is defined in the yaml file, then the default version attribute of rougailconfig is taken
+- if a version attribute is defined in the yaml file, this is the one that is taken
+"""
+
+
+def test_validate_default_version():
+    "retrieves the default_dictionary_format_version if no version in the yaml file"
+
+    RougailConfig["dictionaries_dir"] = ["tests/data/dict1"]
+    RougailConfig["default_dictionary_format_version"] = "1.1"
+    rougail = Rougail()
+    config = rougail.get_config()
+
+    filename = "tests/data/dict1/dict.yml"
+    with open(filename, encoding="utf8") as file_fh:
+        objects = YAML(typ="safe").load(file_fh)
+    version = rougail.converted.validate_file_version(objects, filename)
+    assert version == RougailConfig["default_dictionary_format_version"]
+
+
+def test_validate_file_version_from_yml():
+    "retrives the yaml file version defined in the yaml file"
+
+    RougailConfig["dictionaries_dir"] = ["tests/data/dict2"]
+    RougailConfig["default_dictionary_format_version"] = "1.1"
+    rougail = Rougail()
+    config = rougail.get_config()
+
+    filename = "tests/data/dict2/dict.yml"
+    with open(filename, encoding="utf8") as file_fh:
+        objects = YAML(typ="safe").load(file_fh)
+    version = rougail.converted.validate_file_version(objects, filename)
+    assert version == "1.0"
+
+
+def test_retrieve_version_from_config():
+
+    RougailConfig["dictionaries_dir"] = ["tests/data/dict2"]
+    RougailConfig["default_dictionary_format_version"] = "1.1"
+    rougail = Rougail()
+    # FIXME replace with rougail.annotator()
+    # rougail.converted.annotator()
+    rougail.get_config()
+    assert rougail.converted.paths._data["rougail.hello"].version == "1.0"
+
+
+# def test_dictionary(test_dir):
+#    assert getcwd() == ORI_DIR
+#    test_dir = join(dico_dirs, test_dir)
+#    launch_test(test_dir, 'dict')
+#    assert getcwd() == ORI_DIR