stove/rougail
3
0
Fork 0
Code Issues 11 Pull requests 1 Projects Releases Wiki Activity

default dictionary yaml format version

Browse source
This commit is contained in:
gwen 2024-07-01 18:04:18 +02:00
parent e2d62211ec
commit a7c862cccd
4 changed files with 220 additions and 131 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
.pre-commit-config.yaml Normal file
View file

@ -0,0 +1,47 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.3.0
hooks:
- id: check-yaml
- id: end-of-file-fixer
- id: trailing-whitespace
- id: check-ast
- id: check-added-large-files
- id: check-json
- id: check-executables-have-shebangs
- id: check-symlinks
- repo: https://github.com/psf/black
rev: 22.10.0
hooks:
- id: black
# - repo: https://github.com/hhatto/autopep8
# rev: v2.0.4
# hooks:
# - id: autopep8
# - repo: https://github.com/pre-commit/mirrors-mypy
# rev: v1.6.1
# hooks:
# - id: mypy
# - repo: https://github.com/PyCQA/pylint
# rev: v3.0.2
# hooks:
# - id: pylint
# - repo: https://github.com/PyCQA/isort
# rev: 5.11.5
# hooks:
# - id: isort
# - repo: https://github.com/motet-a/jinjalint
# rev: 0.5
# hooks:
# - id: jinjalint
# - repo: https://github.com/rstcheck/rstcheck
# rev: v6.2.0
# hooks:
# - id: rstcheck

4
src/rougail/config.py
View file

@ -28,14 +28,14 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from os.path import join, abspath, dirname
from os.path import abspath, dirname, join
ROUGAILROOT = "/srv/rougail"
DTDDIR = join(dirname(abspath(__file__)), "data")
RougailConfig = {
"default_dictionary_format_version": None,
"dictionaries_dir": [join(ROUGAILROOT, "dictionaries")],
"extra_dictionaries": {},
"services_dir": [join(ROUGAILROOT, "services")],

280
src/rougail/convert.py
View file

@ -29,46 +29,43 @@ along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
import logging
from itertools import chain
from pathlib import Path
from re import compile, findall
from typing import (
Optional,
Union,
get_type_hints,
Any,
Literal,
List,
Dict,
Iterator,
List,
Literal,
Optional,
Tuple,
Union,
get_type_hints,
)
from itertools import chain
from re import findall, compile
from pydantic import ValidationError
from ruamel.yaml import YAML
from ruamel.yaml.comments import CommentedMap
from pydantic import ValidationError
from tiramisu.error import display_list
from .i18n import _
from .annotator import SpaceAnnotator
from .tiramisureflector import TiramisuReflector
from .utils import get_realpath
from .error import DictConsistencyError
from .i18n import _
from .object_model import CONVERT_OPTION # Choice,
from .object_model import (
CONVERT_OPTION,
Family,
Dynamic,
Variable,
#Choice,
SymLink,
CALCULATION_TYPES,
Calculation,
VariableCalculation,
PARAM_TYPES,
AnyParam,
Calculation,
Dynamic,
Family,
SymLink,
Variable,
VariableCalculation,
)
from .error import DictConsistencyError
from .tiramisureflector import TiramisuReflector
from .utils import get_realpath
property_types = Union[Literal[True], Calculation]
properties_types = Dict[str, property_types]
@ -102,11 +99,12 @@ class Property:
class Paths:
_regexp_relative = compile(r"^_*\.(.*)$")
def __init__(self,
default_namespace: str,
) -> None:
def __init__(
self,
default_namespace: str,
) -> None:
self._data: Dict[str, Union[Variable, Family]] = {}
self._dynamics: Dict[str: str] = {}
self._dynamics: Dict[str:str] = {}
self.default_namespace = default_namespace
self.path_prefix = None
@ -136,7 +134,7 @@ class Paths:
xmlfiles: List[str],
) -> Any:
suffix = None
if version != '1.0' and self._regexp_relative.search(path):
if version != "1.0" and self._regexp_relative.search(path):
relative, subpath = path.split(".", 1)
relative_len = len(relative)
path_len = current_path.count(".")
@ -145,29 +143,34 @@ class Paths:
else:
path = get_realpath(path, suffix_path)
dynamic = None
if not path in self._data and '{{ suffix }}' not in path:
if not path in self._data and "{{ suffix }}" not in path:
new_path = None
current_path = None
for name in path.split('.'):
for name in path.split("."):
parent_path = current_path
if current_path:
current_path += '.' + name
current_path += "." + name
else:
current_path = name
if current_path in self._data:
if new_path:
new_path += '.' + name
new_path += "." + name
else:
new_path = name
continue
for dynamic_path in self._dynamics:
parent_dynamic, name_dynamic = dynamic_path.rsplit('.', 1)
if version == '1.0' and parent_dynamic == parent_path and name_dynamic.endswith('{{ suffix }}') and name == name_dynamic.replace('{{ suffix }}', ''):
new_path += '.' + name_dynamic
parent_dynamic, name_dynamic = dynamic_path.rsplit(".", 1)
if (
version == "1.0"
and parent_dynamic == parent_path
and name_dynamic.endswith("{{ suffix }}")
and name == name_dynamic.replace("{{ suffix }}", "")
):
new_path += "." + name_dynamic
break
else:
if new_path:
new_path += '.' + name
new_path += "." + name
else:
new_path = name
path = new_path
@ -175,33 +178,36 @@ class Paths:
current_path = None
new_path = current_path
suffixes = []
for name in path.split('.'):
for name in path.split("."):
parent_path = current_path
if current_path:
current_path += '.' + name
current_path += "." + name
else:
current_path = name
#parent_path, name_path = path.rsplit('.', 1)
# parent_path, name_path = path.rsplit('.', 1)
if current_path in self._data:
if new_path:
new_path += '.' + name
new_path += "." + name
else:
new_path = name
continue
for dynamic_path in self._dynamics:
parent_dynamic, name_dynamic = dynamic_path.rsplit('.', 1)
if "{{ suffix }}" not in name_dynamic or parent_path != parent_dynamic:
parent_dynamic, name_dynamic = dynamic_path.rsplit(".", 1)
if (
"{{ suffix }}" not in name_dynamic
or parent_path != parent_dynamic
):
continue
regexp = "^" + name_dynamic.replace("{{ suffix }}", "(.*)")
finded = findall(regexp, name)
if len(finded) != 1 or not finded[0]:
continue
suffixes.append(finded[0])
new_path += '.' + name_dynamic
new_path += "." + name_dynamic
break
else:
if new_path:
new_path += '.' + name
new_path += "." + name
else:
new_path = name
if "{{ suffix }}" in name:
@ -213,9 +219,14 @@ class Paths:
return None, None
option = self._data[path]
option_namespace = option.namespace
if self.default_namespace not in [namespace, option_namespace] and namespace != option_namespace:
msg = _(f'A variable or a family located in the "{option_namespace}" namespace '
f'shall not be used in the "{namespace}" namespace')
if (
self.default_namespace not in [namespace, option_namespace]
and namespace != option_namespace
):
msg = _(
f'A variable or a family located in the "{option_namespace}" namespace '
f'shall not be used in the "{namespace}" namespace'
)
raise DictConsistencyError(msg, 38, xmlfiles)
return option, suffixes
@ -292,7 +303,7 @@ class ParserVariable:
#
self.family = Family
self.dynamic = Dynamic
self.choice = Variable #Choice
self.choice = Variable # Choice
#
self.exclude_imports = []
self.informations = Informations()
@ -309,7 +320,7 @@ class ParserVariable:
self.variable = Variable
hint = get_type_hints(self.dynamic)
# FIXME: only for format 1.0
hint['variable'] = str
hint["variable"] = str
self.family_types = hint["type"].__args__ # pylint: disable=W0201
self.family_attrs = frozenset( # pylint: disable=W0201
set(hint) - {"name", "path", "xmlfiles"} | {"redefine"}
@ -320,7 +331,9 @@ class ParserVariable:
#
hint = get_type_hints(self.variable)
self.variable_types = self.convert_options #hint["type"].__args__ # pylint: disable=W0201
self.variable_types = (
self.convert_options
) # hint["type"].__args__ # pylint: disable=W0201
#
hint = get_type_hints(self.choice)
self.choice_attrs = frozenset( # pylint: disable=W0201
@ -379,7 +392,7 @@ class ParserVariable:
else:
return "variable"
else:
if version == '1.0':
if version == "1.0":
msg = f'Invalid value for the variable "{path}": "{obj}"'
raise DictConsistencyError(msg, 102, [filename])
return "variable"
@ -420,7 +433,7 @@ class ParserVariable:
msg = f'the variable or family name "{name}" is incorrect, it must not starts with "_" character'
raise DictConsistencyError(msg, 16, [filename])
path = f"{subpath}.{name}"
if version == '0.1' and not isinstance(obj, dict) and obj is not None:
if version == "0.1" and not isinstance(obj, dict) and obj is not None:
msg = f'the variable "{path}" has a wrong type "{type(obj)}"'
raise DictConsistencyError(msg, 17, [filename])
typ = self.is_family_or_variable(
@ -460,7 +473,7 @@ class ParserVariable:
first_variable: bool = False,
family_is_leadership: bool = False,
family_is_dynamic: bool = False,
parent_dynamic: Optional[str] = None
parent_dynamic: Optional[str] = None,
) -> None:
"""Parse a family"""
if obj is None:
@ -505,14 +518,14 @@ class ParserVariable:
if self.get_family_or_variable_type(family_obj) == "dynamic":
family_is_dynamic = True
parent_dynamic = path
if version == '1.0' and '{{ suffix }}' not in name:
name += '{{ suffix }}'
path += '{{ suffix }}'
if '{{ suffix }}' not in name:
if version == "1.0" and "{{ suffix }}" not in name:
name += "{{ suffix }}"
path += "{{ suffix }}"
if "{{ suffix }}" not in name:
msg = f'dynamic family name must have "{{{{ suffix }}}}" in his name for "{path}"'
raise DictConsistencyError(msg, 13, [filename])
if version != '1.0' and not family_obj and comment:
family_obj['description'] = comment
if version != "1.0" and not family_obj and comment:
family_obj["description"] = comment
self.add_family(
path,
name,
@ -592,29 +605,41 @@ class ParserVariable:
family_obj = self.dynamic
if version == "1.0":
if "variable" not in family:
raise DictConsistencyError(f'dynamic family must have "variable" attribute for "{path}"', 101, family["xmlfiles"])
if 'dynamic' in family:
raise DictConsistencyError('variable and dynamic cannot be set together in the dynamic family "{path}"', 100, family['xmlfiles'])
family['dynamic'] = {'type': 'variable',
'variable': family['variable'],
'propertyerror': False,
'allow_none': True,
}
del family['variable']
#FIXME only for 1.0
raise DictConsistencyError(
f'dynamic family must have "variable" attribute for "{path}"',
101,
family["xmlfiles"],
)
if "dynamic" in family:
raise DictConsistencyError(
'variable and dynamic cannot be set together in the dynamic family "{path}"',
100,
family["xmlfiles"],
)
family["dynamic"] = {
"type": "variable",
"variable": family["variable"],
"propertyerror": False,
"allow_none": True,
}
del family["variable"]
# FIXME only for 1.0
if "variable" in family:
raise Exception(f'dynamic family must not have "variable" attribute for "{family["path"]}" in {family["xmlfiles"]}')
raise Exception(
f'dynamic family must not have "variable" attribute for "{family["path"]}" in {family["xmlfiles"]}'
)
else:
family_obj = self.family
# convert to Calculation objects
self.parse_parameters(path,
family,
filename,
family_is_dynamic,
False,
version,
typ='family',
)
self.parse_parameters(
path,
family,
filename,
family_is_dynamic,
False,
version,
typ="family",
)
try:
self.paths.add(
path,
@ -651,27 +676,28 @@ class ParserVariable:
parent_dynamic: Optional[str] = None,
) -> None:
"""Parse variable"""
if version == '1.0' or isinstance(obj, dict):
if version == "1.0" or isinstance(obj, dict):
if obj is None:
obj = {}
extra_attrs = set(obj) - self.choice_attrs
else:
extra_attrs = []
obj = {'default': obj}
obj = {"default": obj}
if comment:
obj['description'] = comment
obj["description"] = comment
if extra_attrs:
raise Exception(
f'"{path}" is not a valid variable, there are additional '
f'attributes: "{", ".join(extra_attrs)}"'
)
self.parse_parameters(path,
obj,
filename,
family_is_dynamic,
family_is_leadership is True and first_variable is False,
version,
)
self.parse_parameters(
path,
obj,
filename,
family_is_dynamic,
family_is_leadership is True and first_variable is False,
version,
)
self.parse_params(path, obj)
if path in self.paths:
if "exists" in obj and not obj.pop("exists"):
@ -680,7 +706,11 @@ class ParserVariable:
msg = f'Variable "{path}" already exists'
raise DictConsistencyError(msg, 45, [filename])
self.paths.add(
path, self.paths[path].model_copy(update=obj),family_is_dynamic, parent_dynamic, force=True
path,
self.paths[path].model_copy(update=obj),
family_is_dynamic,
parent_dynamic,
force=True,
)
self.paths[path].xmlfiles.append(filename)
else:
@ -694,12 +724,7 @@ class ParserVariable:
raise DictConsistencyError(msg, 46, [filename])
obj["path"] = path
self.add_variable(
name,
obj,
filename,
family_is_dynamic,
parent_dynamic,
version
name, obj, filename, family_is_dynamic, parent_dynamic, version
)
if family_is_leadership:
if first_variable:
@ -707,18 +732,19 @@ class ParserVariable:
else:
self.followers.append(path)
def parse_parameters(self,
path: str,
obj: dict,
filename: str,
family_is_dynamic: bool,
is_follower: bool,
version: str,
*,
typ: str='variable',
):
def parse_parameters(
self,
path: str,
obj: dict,
filename: str,
family_is_dynamic: bool,
is_follower: bool,
version: str,
*,
typ: str = "variable",
):
"""Parse variable or family parameters"""
if typ == 'variable':
if typ == "variable":
calculations = self.choice_calculations
else:
calculations = self.family_calculations
@ -783,7 +809,18 @@ class ParserVariable:
params = []
for key, val in obj["params"].items():
try:
params.append(AnyParam(key=key, value=val, type="any", path=None, is_follower=None, attribute=None, family_is_dynamic=None, xmlfiles=None))
params.append(
AnyParam(
key=key,
value=val,
type="any",
path=None,
is_follower=None,
attribute=None,
family_is_dynamic=None,
xmlfiles=None,
)
)
except ValidationError as err:
raise Exception(
f'"{key}" has an invalid "params" for {path}: {err}'
@ -797,7 +834,7 @@ class ParserVariable:
filename: str,
family_is_dynamic: bool,
parent_dynamic: Optional[str],
version: str
version: str,
) -> None:
"""Add a new variable"""
if not isinstance(filename, list):
@ -916,11 +953,11 @@ class ParserVariable:
else:
param_typ = val["type"]
val["key"] = key
val['path'] = path
val['family_is_dynamic'] = family_is_dynamic
val['is_follower'] = is_follower
val['attribute'] = attribute
val['xmlfiles'] = xmlfiles
val["path"] = path
val["family_is_dynamic"] = family_is_dynamic
val["is_follower"] = is_follower
val["attribute"] = attribute
val["xmlfiles"] = xmlfiles
try:
params.append(PARAM_TYPES[param_typ](**val))
except ValidationError as err:
@ -1031,10 +1068,11 @@ class RougailConvert(ParserVariable):
if path_prefix:
self.path_prefix = None
def get_comment(self,
name: str,
objects: CommentedMap,
) -> Optional[str]:
def get_comment(
self,
name: str,
objects: CommentedMap,
) -> Optional[str]:
if name in objects.ca.items:
comment = objects.ca.items[name][2]
else:
@ -1110,8 +1148,14 @@ class RougailConvert(ParserVariable):
version = str(obj.pop(name))
break
else:
msg = '"version" attribut is mandatory in YAML file'
raise DictConsistencyError(msg, 27, [filename])
# the `version` attribute is not mandatory
default_version = self.rougailconfig["default_dictionary_format_version"]
if default_version is not None:
version = default_version
else:
msg = '"version" attribut is mandatory in YAML file'
raise DictConsistencyError(msg, 27, [filename])
if version not in self.supported_version:
msg = f'version "{version}" is not supported, list of supported versions: {display_list(self.supported_version, separator="or", add_quote=True)}'
raise DictConsistencyError(msg, 28, [filename])

20
src/rougail/update.py
View file

@ -23,29 +23,27 @@ along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from typing import List, Any, Optional, Tuple
from os.path import join, isfile, isdir, basename
from os import listdir, makedirs
from os.path import basename, isdir, isfile, join
from typing import Any, List, Optional, Tuple
try:
from lxml.etree import parse, XMLParser, XMLSyntaxError # pylint: disable=E0611
from lxml.etree import Element, SubElement, tostring
from lxml.etree import SubElement # pylint: disable=E0611
from lxml.etree import Element, XMLParser, XMLSyntaxError, parse, tostring
except ModuleNotFoundError as err:
parse = None
# from ast import parse as ast_parse
from json import dumps
from ruamel.yaml import YAML
from yaml import dump, SafeDumper
from pathlib import Path
from .i18n import _
from .error import UpgradeError
from ruamel.yaml import YAML
from .utils import normalize_family
from .config import RougailConfig
from .error import UpgradeError
from .i18n import _
from .object_model import CONVERT_OPTION
from .utils import normalize_family
VERSIONS = ["0.10", "1.0", "1.1"]
@ -644,7 +642,7 @@ class RougailUpgrade:
ext = "xml"
else:
with xmlsrc.open() as xml_fh:
root = YAML(typ='safe').load(file_fh)
root = YAML(typ="safe").load(file_fh)
search_function_name = get_function_name(str(root["version"]))
ext = "yml"
function_found = False