stove/rougail
3
0
Fork 0
Code Issues 10 Pull requests 1 Projects Releases Wiki Activity

Merge branch 'float_version' into develop

Browse source
This commit is contained in:
gwen 2024-07-09 10:03:16 +02:00
commit d5661511da
9 changed files with 361 additions and 133 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
.pre-commit-config.yaml Normal file
View file

@ -0,0 +1,47 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.3.0
hooks:
- id: check-yaml
- id: end-of-file-fixer
- id: trailing-whitespace
- id: check-ast
- id: check-added-large-files
- id: check-json
- id: check-executables-have-shebangs
- id: check-symlinks
- repo: https://github.com/psf/black
rev: 22.10.0
hooks:
- id: black
# - repo: https://github.com/hhatto/autopep8
# rev: v2.0.4
# hooks:
# - id: autopep8
# - repo: https://github.com/pre-commit/mirrors-mypy
# rev: v1.6.1
# hooks:
# - id: mypy
# - repo: https://github.com/PyCQA/pylint
# rev: v3.0.2
# hooks:
# - id: pylint
# - repo: https://github.com/PyCQA/isort
# rev: 5.11.5
# hooks:
# - id: isort
# - repo: https://github.com/motet-a/jinjalint
# rev: 0.5
# hooks:
# - id: jinjalint
# - repo: https://github.com/rstcheck/rstcheck
# rev: v6.2.0
# hooks:
# - id: rstcheck

44
docs/developer.rst
View file

@ -14,3 +14,47 @@ This process describes how to install and run the project locally, e.g. for deve
*Nota*: command is to be executed through the terminal
`pip install rougail`
Code quality
---------------
We are using `pre-commit <https://pre-commit.com/>`_, there is a :file:`.pre-commit-config.yaml`
pre-commit config file in the root's project.
You need to:
- install the pre-commit library::
pip install pre-commit
- registrer the pre-commit git hooks with this command::
pre-commit install
- launch the quality code procedure with::
pre-commit
or simply just commit your changes, pre-commit will automatically be launched.
.. attention:: If an error is found, the commit will not happen.
You must resolve all errors that pre-commit that pre-commit points out to you before.
.. note:: If you need for some reason to disable `pre-commit`, just set
the `PRE_COMMIT_ALLOW_NO_CONFIG` environment variable before commiting::
PRE_COMMIT_ALLOW_NO_CONFIG=1 git commit
Coding standard
------------------
We use black
.. code-block:: yaml
- repo: https://github.com/psf/black
rev: 22.10.0
hooks:
- id: black
And some YAML and JSON validators.

7
pyproject.toml
View file

@ -26,11 +26,14 @@ dependencies = [
"ruamel.yaml ~= 0.17.40",
"pydantic ~= 2.5.2",
"jinja2 ~= 3.1.2",
"tiramisu ~= 4.1.0",
"tiramisu ~= 4.1.0"
]
[project.optional-dependancies]
[project.optional-dependencies]
dev = [
"pylint ~= 3.0.3",
"pytest ~= 8.2.2",
"lxml ~= 5.2.2"
]
[tool.commitizen]

4
src/rougail/config.py
View file

@ -28,14 +28,14 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from os.path import join, abspath, dirname
from os.path import abspath, dirname, join
ROUGAILROOT = "/srv/rougail"
DTDDIR = join(dirname(abspath(__file__)), "data")
RougailConfig = {
"default_dictionary_format_version": None,
"dictionaries_dir": [join(ROUGAILROOT, "dictionaries")],
"extra_dictionaries": {},
"services_dir": [join(ROUGAILROOT, "services")],

216
src/rougail/convert.py
View file

@ -29,46 +29,43 @@ along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
import logging
from itertools import chain
from pathlib import Path
from re import compile, findall
from typing import (
Optional,
Union,
get_type_hints,
Any,
Literal,
List,
Dict,
Iterator,
List,
Literal,
Optional,
Tuple,
Union,
get_type_hints,
)
from itertools import chain
from re import findall, compile
from pydantic import ValidationError
from ruamel.yaml import YAML
from ruamel.yaml.comments import CommentedMap
from pydantic import ValidationError
from tiramisu.error import display_list
from .i18n import _
from .annotator import SpaceAnnotator
from .tiramisureflector import TiramisuReflector
from .utils import get_realpath
from .error import DictConsistencyError
from .i18n import _
from .object_model import CONVERT_OPTION # Choice,
from .object_model import (
CONVERT_OPTION,
Family,
Dynamic,
Variable,
#Choice,
SymLink,
CALCULATION_TYPES,
Calculation,
VariableCalculation,
PARAM_TYPES,
AnyParam,
Calculation,
Dynamic,
Family,
SymLink,
Variable,
VariableCalculation,
)
from .error import DictConsistencyError
from .tiramisureflector import TiramisuReflector
from .utils import get_realpath
property_types = Union[Literal[True], Calculation]
properties_types = Dict[str, property_types]
@ -102,7 +99,8 @@ class Property:
class Paths:
_regexp_relative = compile(r"^_*\.(.*)$")
def __init__(self,
def __init__(
self,
default_namespace: str,
) -> None:
self._data: Dict[str, Union[Variable, Family]] = {}
@ -136,7 +134,7 @@ class Paths:
xmlfiles: List[str],
) -> Any:
suffix = None
if version != '1.0' and self._regexp_relative.search(path):
if version != "1.0" and self._regexp_relative.search(path):
relative, subpath = path.split(".", 1)
relative_len = len(relative)
path_len = current_path.count(".")
@ -145,29 +143,34 @@ class Paths:
else:
path = get_realpath(path, suffix_path)
dynamic = None
if not path in self._data and '{{ suffix }}' not in path:
if not path in self._data and "{{ suffix }}" not in path:
new_path = None
current_path = None
for name in path.split('.'):
for name in path.split("."):
parent_path = current_path
if current_path:
current_path += '.' + name
current_path += "." + name
else:
current_path = name
if current_path in self._data:
if new_path:
new_path += '.' + name
new_path += "." + name
else:
new_path = name
continue
for dynamic_path in self._dynamics:
parent_dynamic, name_dynamic = dynamic_path.rsplit('.', 1)
if version == '1.0' and parent_dynamic == parent_path and name_dynamic.endswith('{{ suffix }}') and name == name_dynamic.replace('{{ suffix }}', ''):
new_path += '.' + name_dynamic
parent_dynamic, name_dynamic = dynamic_path.rsplit(".", 1)
if (
version == "1.0"
and parent_dynamic == parent_path
and name_dynamic.endswith("{{ suffix }}")
and name == name_dynamic.replace("{{ suffix }}", "")
):
new_path += "." + name_dynamic
break
else:
if new_path:
new_path += '.' + name
new_path += "." + name
else:
new_path = name
path = new_path
@ -175,33 +178,36 @@ class Paths:
current_path = None
new_path = current_path
suffixes = []
for name in path.split('.'):
for name in path.split("."):
parent_path = current_path
if current_path:
current_path += '.' + name
current_path += "." + name
else:
current_path = name
# parent_path, name_path = path.rsplit('.', 1)
if current_path in self._data:
if new_path:
new_path += '.' + name
new_path += "." + name
else:
new_path = name
continue
for dynamic_path in self._dynamics:
parent_dynamic, name_dynamic = dynamic_path.rsplit('.', 1)
if "{{ suffix }}" not in name_dynamic or parent_path != parent_dynamic:
parent_dynamic, name_dynamic = dynamic_path.rsplit(".", 1)
if (
"{{ suffix }}" not in name_dynamic
or parent_path != parent_dynamic
):
continue
regexp = "^" + name_dynamic.replace("{{ suffix }}", "(.*)")
finded = findall(regexp, name)
if len(finded) != 1 or not finded[0]:
continue
suffixes.append(finded[0])
new_path += '.' + name_dynamic
new_path += "." + name_dynamic
break
else:
if new_path:
new_path += '.' + name
new_path += "." + name
else:
new_path = name
if "{{ suffix }}" in name:
@ -213,9 +219,14 @@ class Paths:
return None, None
option = self._data[path]
option_namespace = option.namespace
if self.default_namespace not in [namespace, option_namespace] and namespace != option_namespace:
msg = _(f'A variable or a family located in the "{option_namespace}" namespace '
f'shall not be used in the "{namespace}" namespace')
if (
self.default_namespace not in [namespace, option_namespace]
and namespace != option_namespace
):
msg = _(
f'A variable or a family located in the "{option_namespace}" namespace '
f'shall not be used in the "{namespace}" namespace'
)
raise DictConsistencyError(msg, 38, xmlfiles)
return option, suffixes
@ -309,7 +320,7 @@ class ParserVariable:
self.variable = Variable
hint = get_type_hints(self.dynamic)
# FIXME: only for format 1.0
hint['variable'] = str
hint["variable"] = str
self.family_types = hint["type"].__args__ # pylint: disable=W0201
self.family_attrs = frozenset( # pylint: disable=W0201
set(hint) - {"name", "path", "xmlfiles"} | {"redefine"}
@ -320,7 +331,9 @@ class ParserVariable:
#
hint = get_type_hints(self.variable)
self.variable_types = self.convert_options #hint["type"].__args__ # pylint: disable=W0201
self.variable_types = (
self.convert_options
) # hint["type"].__args__ # pylint: disable=W0201
#
hint = get_type_hints(self.choice)
self.choice_attrs = frozenset( # pylint: disable=W0201
@ -379,7 +392,7 @@ class ParserVariable:
else:
return "variable"
else:
if version == '1.0':
if version == "1.0":
msg = f'Invalid value for the variable "{path}": "{obj}"'
raise DictConsistencyError(msg, 102, [filename])
return "variable"
@ -420,7 +433,7 @@ class ParserVariable:
msg = f'the variable or family name "{name}" is incorrect, it must not starts with "_" character'
raise DictConsistencyError(msg, 16, [filename])
path = f"{subpath}.{name}"
if version == '0.1' and not isinstance(obj, dict) and obj is not None:
if version == "0.1" and not isinstance(obj, dict) and obj is not None:
msg = f'the variable "{path}" has a wrong type "{type(obj)}"'
raise DictConsistencyError(msg, 17, [filename])
typ = self.is_family_or_variable(
@ -460,7 +473,7 @@ class ParserVariable:
first_variable: bool = False,
family_is_leadership: bool = False,
family_is_dynamic: bool = False,
parent_dynamic: Optional[str] = None
parent_dynamic: Optional[str] = None,
) -> None:
"""Parse a family"""
if obj is None:
@ -505,14 +518,14 @@ class ParserVariable:
if self.get_family_or_variable_type(family_obj) == "dynamic":
family_is_dynamic = True
parent_dynamic = path
if version == '1.0' and '{{ suffix }}' not in name:
name += '{{ suffix }}'
path += '{{ suffix }}'
if '{{ suffix }}' not in name:
if version == "1.0" and "{{ suffix }}" not in name:
name += "{{ suffix }}"
path += "{{ suffix }}"
if "{{ suffix }}" not in name:
msg = f'dynamic family name must have "{{{{ suffix }}}}" in his name for "{path}"'
raise DictConsistencyError(msg, 13, [filename])
if version != '1.0' and not family_obj and comment:
family_obj['description'] = comment
if version != "1.0" and not family_obj and comment:
family_obj["description"] = comment
self.add_family(
path,
name,
@ -592,28 +605,40 @@ class ParserVariable:
family_obj = self.dynamic
if version == "1.0":
if "variable" not in family:
raise DictConsistencyError(f'dynamic family must have "variable" attribute for "{path}"', 101, family["xmlfiles"])
if 'dynamic' in family:
raise DictConsistencyError('variable and dynamic cannot be set together in the dynamic family "{path}"', 100, family['xmlfiles'])
family['dynamic'] = {'type': 'variable',
'variable': family['variable'],
'propertyerror': False,
'allow_none': True,
raise DictConsistencyError(
f'dynamic family must have "variable" attribute for "{path}"',
101,
family["xmlfiles"],
)
if "dynamic" in family:
raise DictConsistencyError(
'variable and dynamic cannot be set together in the dynamic family "{path}"',
100,
family["xmlfiles"],
)
family["dynamic"] = {
"type": "variable",
"variable": family["variable"],
"propertyerror": False,
"allow_none": True,
}
del family['variable']
del family["variable"]
# FIXME only for 1.0
if "variable" in family:
raise Exception(f'dynamic family must not have "variable" attribute for "{family["path"]}" in {family["xmlfiles"]}')
raise Exception(
f'dynamic family must not have "variable" attribute for "{family["path"]}" in {family["xmlfiles"]}'
)
else:
family_obj = self.family
# convert to Calculation objects
self.parse_parameters(path,
self.parse_parameters(
path,
family,
filename,
family_is_dynamic,
False,
version,
typ='family',
typ="family",
)
try:
self.paths.add(
@ -651,21 +676,22 @@ class ParserVariable:
parent_dynamic: Optional[str] = None,
) -> None:
"""Parse variable"""
if version == '1.0' or isinstance(obj, dict):
if version == "1.0" or isinstance(obj, dict):
if obj is None:
obj = {}
extra_attrs = set(obj) - self.choice_attrs
else:
extra_attrs = []
obj = {'default': obj}
obj = {"default": obj}
if comment:
obj['description'] = comment
obj["description"] = comment
if extra_attrs:
raise Exception(
f'"{path}" is not a valid variable, there are additional '
f'attributes: "{", ".join(extra_attrs)}"'
)
self.parse_parameters(path,
self.parse_parameters(
path,
obj,
filename,
family_is_dynamic,
@ -680,7 +706,11 @@ class ParserVariable:
msg = f'Variable "{path}" already exists'
raise DictConsistencyError(msg, 45, [filename])
self.paths.add(
path, self.paths[path].model_copy(update=obj),family_is_dynamic, parent_dynamic, force=True
path,
self.paths[path].model_copy(update=obj),
family_is_dynamic,
parent_dynamic,
force=True,
)
self.paths[path].xmlfiles.append(filename)
else:
@ -694,12 +724,7 @@ class ParserVariable:
raise DictConsistencyError(msg, 46, [filename])
obj["path"] = path
self.add_variable(
name,
obj,
filename,
family_is_dynamic,
parent_dynamic,
version
name, obj, filename, family_is_dynamic, parent_dynamic, version
)
if family_is_leadership:
if first_variable:
@ -707,7 +732,8 @@ class ParserVariable:
else:
self.followers.append(path)
def parse_parameters(self,
def parse_parameters(
self,
path: str,
obj: dict,
filename: str,
@ -715,10 +741,10 @@ class ParserVariable:
is_follower: bool,
version: str,
*,
typ: str='variable',
typ: str = "variable",
):
"""Parse variable or family parameters"""
if typ == 'variable':
if typ == "variable":
calculations = self.choice_calculations
else:
calculations = self.family_calculations
@ -783,7 +809,18 @@ class ParserVariable:
params = []
for key, val in obj["params"].items():
try:
params.append(AnyParam(key=key, value=val, type="any", path=None, is_follower=None, attribute=None, family_is_dynamic=None, xmlfiles=None))
params.append(
AnyParam(
key=key,
value=val,
type="any",
path=None,
is_follower=None,
attribute=None,
family_is_dynamic=None,
xmlfiles=None,
)
)
except ValidationError as err:
raise Exception(
f'"{key}" has an invalid "params" for {path}: {err}'
@ -797,7 +834,7 @@ class ParserVariable:
filename: str,
family_is_dynamic: bool,
parent_dynamic: Optional[str],
version: str
version: str,
) -> None:
"""Add a new variable"""
if not isinstance(filename, list):
@ -916,11 +953,11 @@ class ParserVariable:
else:
param_typ = val["type"]
val["key"] = key
val['path'] = path
val['family_is_dynamic'] = family_is_dynamic
val['is_follower'] = is_follower
val['attribute'] = attribute
val['xmlfiles'] = xmlfiles
val["path"] = path
val["family_is_dynamic"] = family_is_dynamic
val["is_follower"] = is_follower
val["attribute"] = attribute
val["xmlfiles"] = xmlfiles
try:
params.append(PARAM_TYPES[param_typ](**val))
except ValidationError as err:
@ -1031,7 +1068,8 @@ class RougailConvert(ParserVariable):
if path_prefix:
self.path_prefix = None
def get_comment(self,
def get_comment(
self,
name: str,
objects: CommentedMap,
) -> Optional[str]:
@ -1109,9 +1147,15 @@ class RougailConvert(ParserVariable):
continue
version = str(obj.pop(name))
break
else:
# the `version` attribute is not mandatory
default_version = self.rougailconfig["default_dictionary_format_version"]
if default_version is not None:
version = default_version
else:
msg = '"version" attribut is mandatory in YAML file'
raise DictConsistencyError(msg, 27, [filename])
if version not in self.supported_version:
msg = f'version "{version}" is not supported, list of supported versions: {display_list(self.supported_version, separator="or", add_quote=True)}'
raise DictConsistencyError(msg, 28, [filename])

20
src/rougail/update.py
View file

@ -23,29 +23,27 @@ along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from typing import List, Any, Optional, Tuple
from os.path import join, isfile, isdir, basename
from os import listdir, makedirs
from os.path import basename, isdir, isfile, join
from typing import Any, List, Optional, Tuple
try:
from lxml.etree import parse, XMLParser, XMLSyntaxError # pylint: disable=E0611
from lxml.etree import Element, SubElement, tostring
from lxml.etree import SubElement # pylint: disable=E0611
from lxml.etree import Element, XMLParser, XMLSyntaxError, parse, tostring
except ModuleNotFoundError as err:
parse = None
# from ast import parse as ast_parse
from json import dumps
from ruamel.yaml import YAML
from yaml import dump, SafeDumper
from pathlib import Path
from .i18n import _
from .error import UpgradeError
from ruamel.yaml import YAML
from .utils import normalize_family
from .config import RougailConfig
from .error import UpgradeError
from .i18n import _
from .object_model import CONVERT_OPTION
from .utils import normalize_family
VERSIONS = ["0.10", "1.0", "1.1"]
@ -644,7 +642,7 @@ class RougailUpgrade:
ext = "xml"
else:
with xmlsrc.open() as xml_fh:
root = YAML(typ='safe').load(file_fh)
root = YAML(typ="safe").load(file_fh)
search_function_name = get_function_name(str(root["version"]))
ext = "yml"
function_found = False

5
tests/data/dict1/dict.yml Normal file
View file

@ -0,0 +1,5 @@
# dict with a correct version declared
#version: "1.1"
hello:
type: string
default: world

5
tests/data/dict2/dict.yml Normal file
View file

@ -0,0 +1,5 @@
# dict with a correct version declared
version: "1.0"
hello:
type: string
default: world

82
tests/test_version.py Normal file
View file

@ -0,0 +1,82 @@
from shutil import rmtree # , copyfile, copytree
from os import getcwd, makedirs
from os.path import isfile, join, isdir
from pytest import fixture, raises
from os import listdir
from rougail import Rougail, RougailConfig
from rougail.error import DictConsistencyError
from ruamel.yaml import YAML
# dico_dirs = 'tests/data'
# test_ok = set()
# for test in listdir(dico_dirs):
# if isdir(join(dico_dirs, test)):
# test_ok.add(test)
# excludes = set([])
# test_ok -= excludes
# ORI_DIR = getcwd()
# test_ok = list(test_ok)
# test_ok.sort()
##print(test_ok)
# @fixture(scope="module", params=test_ok)
# def test_dir(request):
# return request.param
"""
the kinematics are as follows:
- if no version attribute is defined in the yaml file, then the default version attribute of rougailconfig is taken
- if a version attribute is defined in the yaml file, this is the one that is taken
"""
def test_validate_default_version():
"retrieves the default_dictionary_format_version if no version in the yaml file"
RougailConfig["dictionaries_dir"] = ["tests/data/dict1"]
RougailConfig["default_dictionary_format_version"] = "1.1"
rougail = Rougail()
config = rougail.get_config()
filename = "tests/data/dict1/dict.yml"
with open(filename, encoding="utf8") as file_fh:
objects = YAML(typ="safe").load(file_fh)
version = rougail.converted.validate_file_version(objects, filename)
assert version == RougailConfig["default_dictionary_format_version"]
def test_validate_file_version_from_yml():
"retrives the yaml file version defined in the yaml file"
RougailConfig["dictionaries_dir"] = ["tests/data/dict2"]
RougailConfig["default_dictionary_format_version"] = "1.1"
rougail = Rougail()
config = rougail.get_config()
filename = "tests/data/dict2/dict.yml"
with open(filename, encoding="utf8") as file_fh:
objects = YAML(typ="safe").load(file_fh)
version = rougail.converted.validate_file_version(objects, filename)
assert version == "1.0"
def test_retrieve_version_from_config():
RougailConfig["dictionaries_dir"] = ["tests/data/dict2"]
RougailConfig["default_dictionary_format_version"] = "1.1"
rougail = Rougail()
# FIXME replace with rougail.annotator()
# rougail.converted.annotator()
rougail.get_config()
assert rougail.converted.paths._data["rougail.hello"].version == "1.0"
# def test_dictionary(test_dir):
# assert getcwd() == ORI_DIR
# test_dir = join(dico_dirs, test_dir)
# launch_test(test_dir, 'dict')
# assert getcwd() == ORI_DIR