stove/rougail
3
0
Fork 0
Code Issues 10 Pull requests 1 Projects Releases Wiki Activity

feat: can sort dictionaries in different directories

Browse source
This commit is contained in:
egarette@silique.fr 2024-10-25 08:13:52 +02:00
parent 4da9a50265
commit 609914893b
4 changed files with 288 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

203
src/rougail/__init__.py
View file

@ -27,10 +27,11 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
""" """
from tiramisu import Config from tiramisu import Config, undefined
from tiramisu.error import PropertiesOptionError from tiramisu.error import PropertiesOptionError, LeadershipError, ConfigError
from warnings import warn from warnings import warn
from typing import List from typing import List
from re import compile, findall
from .convert import RougailConvert from .convert import RougailConvert
from .config import RougailConfig from .config import RougailConfig
@ -39,15 +40,20 @@ from .object_model import CONVERT_OPTION
from .utils import normalize_family from .utils import normalize_family
def tiramisu_display_name(kls, subconfig) -> str: def tiramisu_display_name(kls,
subconfig,
with_quote: bool=False,
) -> str:
"""Replace the Tiramisu display_name function to display path + description""" """Replace the Tiramisu display_name function to display path + description"""
doc = kls._get_information(subconfig, "doc", None) doc = kls._get_information(subconfig, "doc", None)
comment = f" ({doc})" if doc and doc != kls.impl_getname() else "" comment = f" ({doc})" if doc and doc != kls.impl_getname() else ""
if "{{ suffix }}" in comment: if "{{ identifier }}" in comment:
comment = comment.replace('{{ suffix }}', str(subconfig.suffixes[-1])) comment = comment.replace('{{ identifier }}', str(subconfig.identifiers[-1]))
path = kls.impl_getpath() path = kls.impl_getpath()
if "{{ suffix }}" in path: if "{{ identifier }}" in path and subconfig.identifiers:
path = path.replace('{{ suffix }}', normalize_family(str(subconfig.suffixes[-1]))) path = path.replace('{{ identifier }}', normalize_family(str(subconfig.identifiers[-1])))
if with_quote:
return f'"{path}"{comment}'
return f"{path}{comment}" return f"{path}{comment}"
@ -96,32 +102,71 @@ class Rougail:
errors = [] errors = []
warnings = [] warnings = []
for datas in user_datas: for datas in user_datas:
options = datas.get('options', {})
for name, data in datas.get('values', {}).items(): for name, data in datas.get('values', {}).items():
values.setdefault(name, {}).update(data) values[name] = {'values': data,
'options': options.copy(),
}
errors.extend(datas.get('errors', [])) errors.extend(datas.get('errors', []))
warnings.extend(datas.get('warnings', [])) warnings.extend(datas.get('warnings', []))
self._auto_configure_dynamics(values)
while values: while values:
value_is_set = False value_is_set = False
for option in self.config: for option in self._get_variable(self.config):
if option.path() in values and option.index() in values[option.path()]: path = option.path()
if path not in values:
path = path.upper()
options = values.get(path, {}).get('options', {})
if path not in values or options.get('upper') is not True:
continue
else:
options = values[path].get('options', {})
value = values[path]["values"]
if option.ismulti():
if options.get('multi_separator') and not isinstance(value, list):
value = value.split(options['multi_separator'])
values[path]["values"] = value
if options.get('needs_convert'):
value = [convert_value(option, val) for val in value]
values[path]["values"] = value
values[path]["options"]["needs_convert"] = False
elif options.get('needs_convert'):
value = convert_value(option, value)
index = option.index()
if index is not None:
if not isinstance(value, list) or index >= len(value):
continue
value = value[index]
try: try:
option.value.set(values[option.path()]) option.value.set(value)
value_is_set = True value_is_set = True
values.pop(option.path()) if index is not None:
except: values[path]['values'][index] = undefined
pass if set(values[path]['values']) == {undefined}:
values.pop(path)
else:
values.pop(path)
except Exception as err:
if path != option.path():
values[option.path()] = values.pop(path)
if not value_is_set: if not value_is_set:
break break
for path, data in values.items(): for path, data in values.items():
for index, value in data.items():
try: try:
print('attention', path, value) option = self.config.option(path)
self.config.option(path).value.set(value) value = data['values']
print('pfff') if option.isfollower():
for index, val in enumerate(value):
if val is undefined:
continue
self.config.option(path, index).value.set(val)
else:
option.value.set(value)
except AttributeError as err: except AttributeError as err:
errors.append(str(err)) errors.append(str(err))
except ValueError as err: except (ValueError, LeadershipError) as err:
errors.append(str(err).replace('"', "'")) #errors.append(str(err).replace('"', "'"))
errors.append(str(err))
except PropertiesOptionError as err: except PropertiesOptionError as err:
# warnings.append(f'"{err}" but is defined in "{self.filename}"') # warnings.append(f'"{err}" but is defined in "{self.filename}"')
warnings.append(str(err)) warnings.append(str(err))
@ -129,4 +174,122 @@ class Rougail:
'warnings': warnings, 'warnings': warnings,
} }
def _get_variable(self, config):
for subconfig in config:
if subconfig.isoptiondescription():
yield from self._get_variable(subconfig)
else:
yield subconfig
def _auto_configure_dynamics(self,
values,
):
cache = {}
added = []
for path, data in list(values.items()):
value = data['values']
# for value in data['values'].items():
try:
option = self.config.option(path)
option.name()
except (ConfigError, PropertiesOptionError):
pass
except AttributeError:
config = self.config
current_path = ''
identifiers = []
for name in path.split('.')[:-1]:
if current_path:
current_path += '.'
current_path += name
if current_path in cache:
config, identifier = cache[current_path]
identifiers.append(identifier)
else:
tconfig = config.option(name)
try:
tconfig.group_type()
config = tconfig
except AttributeError:
for tconfig in config.list(uncalculated=True):
if tconfig.isdynamic(only_self=True):
identifier = self._get_identifier(tconfig.name(), name)
if identifier is None:
continue
dynamic_variable = tconfig.information.get('dynamic_variable',
None,
)
if not dynamic_variable:
continue
option_type = self.config.option(dynamic_variable).information.get('type')
if identifiers:
for s in identifiers:
dynamic_variable = dynamic_variable.replace('{{ identifier }}', str(s), 1)
if dynamic_variable not in values:
values[dynamic_variable] = {'values': []}
added.append(dynamic_variable)
elif dynamic_variable not in added:
continue
config = tconfig
# option_type = option.information.get('type')
typ = CONVERT_OPTION.get(option_type, {}).get("func")
if typ:
identifier = typ(identifier)
if identifier not in values[dynamic_variable]['values']:
values[dynamic_variable]['values'].append(identifier)
identifiers.append(identifier)
cache[current_path] = config, identifier
break
else:
if option.isdynamic():
parent_option = self.config.option(path.rsplit('.', 1)[0])
identifiers = self._get_identifier(parent_option.name(uncalculated=True),
parent_option.name(),
)
dynamic_variable = None
while True:
dynamic_variable = parent_option.information.get('dynamic_variable',
None,
)
if dynamic_variable:
break
parent_option = self.config.option(parent_option.path().rsplit('.', 1)[0])
if '.' not in parent_option.path():
parent_option = None
break
if not parent_option:
continue
identifiers = parent_option.identifiers()
for identifier in identifiers:
dynamic_variable = dynamic_variable.replace('{{ identifier }}', str(identifier), 1)
if dynamic_variable not in values:
values[dynamic_variable] = {'values': []}
added.append(dynamic_variable)
elif dynamic_variable not in added:
continue
option_type = option.information.get('type')
typ = CONVERT_OPTION.get(option_type, {}).get("func")
if typ:
identifier = typ(identifier)
if identifier not in values[dynamic_variable]['values']:
values[dynamic_variable]['values'].append(identifier)
cache[option.path()] = option, identifier
def _get_identifier(self, true_name, name) -> str:
regexp = true_name.replace("{{ identifier }}", "(.*)")
finded = findall(regexp, name)
if len(finded) != 1 or not finded[0]:
return
return finded[0]
def convert_value(option, value):
if value == '':
return None
option_type = option.information.get('type')
func = CONVERT_OPTION.get(option_type, {}).get("func")
if func:
return func(value)
return value
__all__ = ("Rougail", "RougailConfig", "RougailUpgrade") __all__ = ("Rougail", "RougailConfig", "RougailUpgrade")

8
src/rougail/annotator/variable.py
View file

@ -178,13 +178,7 @@ class Annotator(Walk): # pylint: disable=R0903
msg = _(f'the variable "{variable.path}" has regexp attribut but has not the "regexp" type') msg = _(f'the variable "{variable.path}" has regexp attribut but has not the "regexp" type')
raise DictConsistencyError(msg, 37, variable.xmlfiles) raise DictConsistencyError(msg, 37, variable.xmlfiles)
if variable.mandatory is None: if variable.mandatory is None:
family_path = variable.path variable.mandatory = True
hidden = variable.hidden is True
while '.' in family_path and not hidden:
family_path = family_path.rsplit(".", 1)[0]
family = self.objectspace.paths[family_path]
hidden = family.hidden is True
variable.mandatory = not hidden
def convert_test(self): def convert_test(self):
"""Convert variable tests value""" """Convert variable tests value"""

35
src/rougail/config.py
View file

@ -193,7 +193,7 @@ def get_rougail_config(*,
main_namespace_default = 'rougail' main_namespace_default = 'rougail'
else: else:
main_namespace_default = 'null' main_namespace_default = 'null'
rougail_options = """default_dictionary_format_version: rougail_options = f"""default_dictionary_format_version:
description: Dictionary format version by default, if not specified in dictionary file description: Dictionary format version by default, if not specified in dictionary file
alternative_name: v alternative_name: v
choices: choices:
@ -219,7 +219,7 @@ sort_dictionaries_all:
main_namespace: main_namespace:
description: Main namespace name description: Main namespace name
default: MAIN_MAMESPACE_DEFAULT default: {main_namespace_default}
alternative_name: s alternative_name: s
mandatory: false mandatory: false
@ -289,18 +289,29 @@ functions_files:
modes_level: modes_level:
description: All modes level available description: All modes level available
multi: true
mandatory: false
"""
if backward_compatibility:
rougail_options += """
default: default:
- basic - basic
- standard - standard
- advanced - advanced
commandline: false """
rougail_options += """
default_family_mode: default_family_mode:
description: Default mode for a family description: Default mode for a family
default: default:
type: jinja
jinja: | jinja: |
{% if modes_level %}
{{ modes_level[0] }} {{ modes_level[0] }}
{% endif %}
disabled:
jinja: |
{% if not modes_level %}
No mode
{% endif %}
validators: validators:
- type: jinja - type: jinja
jinja: | jinja: |
@ -312,9 +323,19 @@ default_family_mode:
default_variable_mode: default_variable_mode:
description: Default mode for a variable description: Default mode for a variable
default: default:
type: jinja
jinja: | jinja: |
{% if modes_level %}
{% if modes_level | length == 1 %}
{{ modes_level[0] }}
{% else %}
{{ modes_level[1] }} {{ modes_level[1] }}
{% endif %}
{% endif %}
disabled:
jinja: |
{% if not modes_level %}
No mode
{% endif %}
validators: validators:
- type: jinja - type: jinja
jinja: | jinja: |
@ -364,7 +385,7 @@ suffix:
default: '' default: ''
mandatory: false mandatory: false
commandline: false commandline: false
""".replace('MAIN_MAMESPACE_DEFAULT', main_namespace_default) """
processes = {'structural': [], processes = {'structural': [],
'output': [], 'output': [],
'user data': [], 'user data': [],

106
src/rougail/convert.py
View file

@ -133,7 +133,7 @@ class Paths:
if not force and is_dynamic: if not force and is_dynamic:
self._dynamics[path] = dynamic self._dynamics[path] = dynamic
def get_relative_path(self, def get_full_path(self,
path: str, path: str,
current_path: str, current_path: str,
): ):
@ -148,21 +148,22 @@ class Paths:
def get_with_dynamic( def get_with_dynamic(
self, self,
path: str, path: str,
suffix_path: str, identifier_path: str,
current_path: str, current_path: str,
version: str, version: str,
namespace: str, namespace: str,
xmlfiles: List[str], xmlfiles: List[str],
) -> Any: ) -> Any:
suffix = None identifier = None
if version != '1.0' and self.regexp_relative.search(path): if version != '1.0' and self.regexp_relative.search(path):
path = self.get_relative_path(path, path = self.get_full_path(path,
current_path, current_path,
) )
else: else:
path = get_realpath(path, suffix_path) path = get_realpath(path, identifier_path)
dynamic = None dynamic = None
if not path in self._data and "{{ suffix }}" not in path: # version 1.0
if version == "1.0" and not path in self._data and "{{ identifier }}" not in path:
new_path = None new_path = None
current_path = None current_path = None
for name in path.split("."): for name in path.split("."):
@ -184,10 +185,9 @@ class Paths:
parent_dynamic = None parent_dynamic = None
name_dynamic = dynamic_path name_dynamic = dynamic_path
if ( if (
version == "1.0" parent_dynamic == parent_path
and parent_dynamic == parent_path and name_dynamic.endswith("{{ identifier }}")
and name_dynamic.endswith("{{ suffix }}") and name == name_dynamic.replace("{{ identifier }}", "")
and name == name_dynamic.replace("{{ suffix }}", "")
): ):
new_path += "." + name_dynamic new_path += "." + name_dynamic
break break
@ -197,12 +197,12 @@ class Paths:
else: else:
new_path = name new_path = name
path = new_path path = new_path
if not path in self._data: if version != "1.0" and not path in self._data:
current_path = None current_path = None
parent_path = None
new_path = current_path new_path = current_path
suffixes = [] identifiers = []
for name in path.split("."): for name in path.split("."):
parent_path = current_path
if current_path: if current_path:
current_path += "." + name current_path += "." + name
else: else:
@ -213,6 +213,7 @@ class Paths:
new_path += "." + name new_path += "." + name
else: else:
new_path = name new_path = name
parent_path = current_path
continue continue
for dynamic_path in self._dynamics: for dynamic_path in self._dynamics:
if '.' in dynamic_path: if '.' in dynamic_path:
@ -221,30 +222,35 @@ class Paths:
parent_dynamic = None parent_dynamic = None
name_dynamic = dynamic_path name_dynamic = dynamic_path
if ( if (
"{{ suffix }}" not in name_dynamic "{{ identifier }}" not in name_dynamic
or parent_path != parent_dynamic or parent_path != parent_dynamic
): ):
continue continue
regexp = "^" + name_dynamic.replace("{{ suffix }}", "(.*)") regexp = "^" + name_dynamic.replace("{{ identifier }}", "(.*)")
finded = findall(regexp, name) finded = findall(regexp, name)
if len(finded) != 1 or not finded[0]: if len(finded) != 1 or not finded[0]:
continue continue
suffixes.append(finded[0]) if finded[0] == "{{ identifier }}":
identifiers.append(None)
else:
identifiers.append(finded[0])
if new_path is None: if new_path is None:
new_path = name_dynamic new_path = name_dynamic
else: else:
new_path += "." + name_dynamic new_path += "." + name_dynamic
parent_path = dynamic_path
break break
else: else:
if new_path: if new_path:
new_path += "." + name new_path += "." + name
else: else:
new_path = name new_path = name
if "{{ suffix }}" in name: if "{{ identifier }}" in name:
suffixes.append(None) identifiers.append(None)
parent_path = current_path
path = new_path path = new_path
else: else:
suffixes = None identifiers = None
if path not in self._data: if path not in self._data:
return None, None return None, None
option = self._data[path] option = self._data[path]
@ -258,7 +264,7 @@ class Paths:
f'shall not be used in the "{namespace}" namespace' f'shall not be used in the "{namespace}" namespace'
) )
raise DictConsistencyError(msg, 38, xmlfiles) raise DictConsistencyError(msg, 38, xmlfiles)
return option, suffixes return option, identifiers
def __getitem__( def __getitem__(
self, self,
@ -316,8 +322,8 @@ class Informations:
class ParserVariable: class ParserVariable:
def __init__(self, rougailconfig): def __init__(self, rougailconfig):
self.load_config(rougailconfig)
self.rougailconfig = rougailconfig self.rougailconfig = rougailconfig
self.load_config()
self.paths = Paths(self.main_namespace) self.paths = Paths(self.main_namespace)
self.families = [] self.families = []
self.variables = [] self.variables = []
@ -342,12 +348,14 @@ class ParserVariable:
self.is_init = False self.is_init = False
super().__init__() super().__init__()
def load_config(self, def load_config(self) -> None:
rougailconfig: 'RougailConfig', rougailconfig = self.rougailconfig
) -> None: self.sort_dictionaries_all = rougailconfig['sort_dictionaries_all']
self.rougailconfig = rougailconfig try:
self.main_namespace = rougailconfig["main_namespace"]
self.main_dictionaries = rougailconfig["main_dictionaries"] self.main_dictionaries = rougailconfig["main_dictionaries"]
except:
self.main_dictionaries = []
self.main_namespace = rougailconfig["main_namespace"]
if self.main_namespace: if self.main_namespace:
self.extra_dictionaries = rougailconfig["extra_dictionaries"] self.extra_dictionaries = rougailconfig["extra_dictionaries"]
self.suffix = rougailconfig["suffix"] self.suffix = rougailconfig["suffix"]
@ -355,6 +363,7 @@ class ParserVariable:
self.custom_types = rougailconfig["custom_types"] self.custom_types = rougailconfig["custom_types"]
self.functions_files = rougailconfig["functions_files"] self.functions_files = rougailconfig["functions_files"]
self.modes_level = rougailconfig["modes_level"] self.modes_level = rougailconfig["modes_level"]
if self.modes_level:
self.default_variable_mode = rougailconfig["default_variable_mode"] self.default_variable_mode = rougailconfig["default_variable_mode"]
self.default_family_mode = rougailconfig["default_family_mode"] self.default_family_mode = rougailconfig["default_family_mode"]
self.extra_annotators = rougailconfig["extra_annotators"] self.extra_annotators = rougailconfig["extra_annotators"]
@ -362,7 +371,7 @@ class ParserVariable:
self.export_with_import = rougailconfig["export_with_import"] self.export_with_import = rougailconfig["export_with_import"]
self.internal_functions = rougailconfig["internal_functions"] self.internal_functions = rougailconfig["internal_functions"]
self.add_extra_options = rougailconfig["structural_commandline.add_extra_options"] self.add_extra_options = rougailconfig["structural_commandline.add_extra_options"]
self.plugins = [] self.plugins = rougailconfig["plugins"]
def _init(self): def _init(self):
if self.is_init: if self.is_init:
@ -557,8 +566,10 @@ class ParserVariable:
# it's just for modify subfamily or subvariable, do not redefine # it's just for modify subfamily or subvariable, do not redefine
if family_obj: if family_obj:
if not obj.pop("redefine", False): if not obj.pop("redefine", False):
raise Exception( raise DictConsistencyError(
f"The family {path} already exists and she is not redefined in {filename}" f'The family "{path}" already exists and it is not redefined',
32,
[filename],
) )
# convert to Calculation objects # convert to Calculation objects
self.parse_parameters( self.parse_parameters(
@ -604,12 +615,12 @@ class ParserVariable:
if obj_type == "dynamic": if obj_type == "dynamic":
family_is_dynamic = True family_is_dynamic = True
parent_dynamic = path parent_dynamic = path
if '{{ suffix }}' not in name: if '{{ identifier }}' not in name:
if "variable" in family_obj: if "variable" in family_obj:
name += '{{ suffix }}' name += '{{ identifier }}'
path += '{{ suffix }}' path += '{{ identifier }}'
else: else:
msg = f'dynamic family name must have "{{{{ suffix }}}}" in his name for "{path}"' msg = f'dynamic family name must have "{{{{ identifier }}}}" in his name for "{path}"'
raise DictConsistencyError(msg, 13, [filename]) raise DictConsistencyError(msg, 13, [filename])
if version != '1.0' and not family_obj and comment: if version != '1.0' and not family_obj and comment:
family_obj['description'] = comment family_obj['description'] = comment
@ -938,6 +949,7 @@ class ParserVariable:
variable["namespace"] = self.namespace variable["namespace"] = self.namespace
variable["version"] = version variable["version"] = version
variable["path_prefix"] = self.path_prefix
variable["xmlfiles"] = filename variable["xmlfiles"] = filename
variable_type = self.get_family_or_variable_type(variable) variable_type = self.get_family_or_variable_type(variable)
obj = { obj = {
@ -1074,8 +1086,8 @@ class ParserVariable:
try: try:
params.append(PARAM_TYPES[param_typ](**val)) params.append(PARAM_TYPES[param_typ](**val))
except ValidationError as err: except ValidationError as err:
raise Exception( raise DictConsistencyError(
f'"{attribute}" has an invalid "{key}" for {path}: {err}' f'"{attribute}" has an invalid "{key}" for {path}: {err}', 29, xmlfiles
) from err ) from err
calculation_object["params"] = params calculation_object["params"] = params
# #
@ -1086,8 +1098,8 @@ class ParserVariable:
f'unknown "return_type" in {attribute} of variable "{path}"' f'unknown "return_type" in {attribute} of variable "{path}"'
) )
# #
if typ == "suffix" and not family_is_dynamic: if typ == "identifier" and not family_is_dynamic:
msg = f'suffix calculation for "{attribute}" in "{path}" cannot be set none dynamic family' msg = f'identifier calculation for "{attribute}" in "{path}" cannot be set none dynamic family'
raise DictConsistencyError(msg, 53, xmlfiles) raise DictConsistencyError(msg, 53, xmlfiles)
if attribute in PROPERTY_ATTRIBUTE: if attribute in PROPERTY_ATTRIBUTE:
calc = CALCULATION_PROPERTY_TYPES[typ](**calculation_object) calc = CALCULATION_PROPERTY_TYPES[typ](**calculation_object)
@ -1143,15 +1155,16 @@ class RougailConvert(ParserVariable):
"""Parse directories content""" """Parse directories content"""
self._init() self._init()
if path_prefix: if path_prefix:
if path_prefix in self.parents: n_path_prefix = normalize_family(path_prefix)
if n_path_prefix in self.parents:
raise Exception("pfffff") raise Exception("pfffff")
root_parent = path_prefix root_parent = n_path_prefix
self.path_prefix = path_prefix self.path_prefix = n_path_prefix
self.namespace = None self.namespace = None
self.add_family( self.add_family(
path_prefix, n_path_prefix,
path_prefix, n_path_prefix,
{}, {'description': path_prefix},
"", "",
False, False,
None, None,
@ -1267,10 +1280,13 @@ class RougailConvert(ParserVariable):
"""Sort filename""" """Sort filename"""
if not isinstance(directories, list): if not isinstance(directories, list):
directories = [directories] directories = [directories]
if self.sort_dictionaries_all:
filenames = {}
for directory_name in directories: for directory_name in directories:
directory = Path(directory_name) directory = Path(directory_name)
if not directory.is_dir(): if not directory.is_dir():
continue continue
if not self.sort_dictionaries_all:
filenames = {} filenames = {}
for file_path in directory.iterdir(): for file_path in directory.iterdir():
if file_path.suffix not in [".yml", ".yaml"]: if file_path.suffix not in [".yml", ".yaml"]:
@ -1282,6 +1298,10 @@ class RougailConvert(ParserVariable):
[filenames[file_path.name][1]], [filenames[file_path.name][1]],
) )
filenames[file_path.name] = str(file_path) filenames[file_path.name] = str(file_path)
if not self.sort_dictionaries_all:
for filename in sorted(filenames):
yield filenames[filename]
if self.sort_dictionaries_all:
for filename in sorted(filenames): for filename in sorted(filenames):
yield filenames[filename] yield filenames[filename]