stove/rougail
3
0
Fork 0
Code Issues 10 Pull requests 1 Projects Releases Wiki Activity

feat: can sort dictionaries in different directories

Browse source
This commit is contained in:
egarette@silique.fr 2024-10-25 08:13:52 +02:00
parent 4da9a50265
commit 609914893b
4 changed files with 288 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

219
src/rougail/__init__.py
View file

@ -27,10 +27,11 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from tiramisu import Config
from tiramisu.error import PropertiesOptionError
from tiramisu import Config, undefined
from tiramisu.error import PropertiesOptionError, LeadershipError, ConfigError
from warnings import warn
from typing import List
from re import compile, findall
from .convert import RougailConvert
from .config import RougailConfig
@ -39,15 +40,20 @@ from .object_model import CONVERT_OPTION
from .utils import normalize_family
def tiramisu_display_name(kls, subconfig) -> str:
def tiramisu_display_name(kls,
subconfig,
with_quote: bool=False,
) -> str:
"""Replace the Tiramisu display_name function to display path + description"""
doc = kls._get_information(subconfig, "doc", None)
comment = f" ({doc})" if doc and doc != kls.impl_getname() else ""
if "{{ suffix }}" in comment:
comment = comment.replace('{{ suffix }}', str(subconfig.suffixes[-1]))
if "{{ identifier }}" in comment:
comment = comment.replace('{{ identifier }}', str(subconfig.identifiers[-1]))
path = kls.impl_getpath()
if "{{ suffix }}" in path:
path = path.replace('{{ suffix }}', normalize_family(str(subconfig.suffixes[-1])))
if "{{ identifier }}" in path and subconfig.identifiers:
path = path.replace('{{ identifier }}', normalize_family(str(subconfig.identifiers[-1])))
if with_quote:
return f'"{path}"{comment}'
return f"{path}{comment}"
@ -96,37 +102,194 @@ class Rougail:
errors = []
warnings = []
for datas in user_datas:
options = datas.get('options', {})
for name, data in datas.get('values', {}).items():
values.setdefault(name, {}).update(data)
values[name] = {'values': data,
'options': options.copy(),
}
errors.extend(datas.get('errors', []))
warnings.extend(datas.get('warnings', []))
self._auto_configure_dynamics(values)
while values:
value_is_set = False
for option in self.config:
if option.path() in values and option.index() in values[option.path()]:
try:
option.value.set(values[option.path()])
value_is_set = True
values.pop(option.path())
except:
pass
for option in self._get_variable(self.config):
path = option.path()
if path not in values:
path = path.upper()
options = values.get(path, {}).get('options', {})
if path not in values or options.get('upper') is not True:
continue
else:
options = values[path].get('options', {})
value = values[path]["values"]
if option.ismulti():
if options.get('multi_separator') and not isinstance(value, list):
value = value.split(options['multi_separator'])
values[path]["values"] = value
if options.get('needs_convert'):
value = [convert_value(option, val) for val in value]
values[path]["values"] = value
values[path]["options"]["needs_convert"] = False
elif options.get('needs_convert'):
value = convert_value(option, value)
index = option.index()
if index is not None:
if not isinstance(value, list) or index >= len(value):
continue
value = value[index]
try:
option.value.set(value)
value_is_set = True
if index is not None:
values[path]['values'][index] = undefined
if set(values[path]['values']) == {undefined}:
values.pop(path)
else:
values.pop(path)
except Exception as err:
if path != option.path():
values[option.path()] = values.pop(path)
if not value_is_set:
break
for path, data in values.items():
for index, value in data.items():
try:
print('attention', path, value)
self.config.option(path).value.set(value)
print('pfff')
except AttributeError as err:
errors.append(str(err))
except ValueError as err:
errors.append(str(err).replace('"', "'"))
except PropertiesOptionError as err:
# warnings.append(f'"{err}" but is defined in "{self.filename}"')
warnings.append(str(err))
try:
option = self.config.option(path)
value = data['values']
if option.isfollower():
for index, val in enumerate(value):
if val is undefined:
continue
self.config.option(path, index).value.set(val)
else:
option.value.set(value)
except AttributeError as err:
errors.append(str(err))
except (ValueError, LeadershipError) as err:
#errors.append(str(err).replace('"', "'"))
errors.append(str(err))
except PropertiesOptionError as err:
# warnings.append(f'"{err}" but is defined in "{self.filename}"')
warnings.append(str(err))
return {'errors': errors,
'warnings': warnings,
}
def _get_variable(self, config):
for subconfig in config:
if subconfig.isoptiondescription():
yield from self._get_variable(subconfig)
else:
yield subconfig
def _auto_configure_dynamics(self,
values,
):
cache = {}
added = []
for path, data in list(values.items()):
value = data['values']
# for value in data['values'].items():
try:
option = self.config.option(path)
option.name()
except (ConfigError, PropertiesOptionError):
pass
except AttributeError:
config = self.config
current_path = ''
identifiers = []
for name in path.split('.')[:-1]:
if current_path:
current_path += '.'
current_path += name
if current_path in cache:
config, identifier = cache[current_path]
identifiers.append(identifier)
else:
tconfig = config.option(name)
try:
tconfig.group_type()
config = tconfig
except AttributeError:
for tconfig in config.list(uncalculated=True):
if tconfig.isdynamic(only_self=True):
identifier = self._get_identifier(tconfig.name(), name)
if identifier is None:
continue
dynamic_variable = tconfig.information.get('dynamic_variable',
None,
)
if not dynamic_variable:
continue
option_type = self.config.option(dynamic_variable).information.get('type')
if identifiers:
for s in identifiers:
dynamic_variable = dynamic_variable.replace('{{ identifier }}', str(s), 1)
if dynamic_variable not in values:
values[dynamic_variable] = {'values': []}
added.append(dynamic_variable)
elif dynamic_variable not in added:
continue
config = tconfig
# option_type = option.information.get('type')
typ = CONVERT_OPTION.get(option_type, {}).get("func")
if typ:
identifier = typ(identifier)
if identifier not in values[dynamic_variable]['values']:
values[dynamic_variable]['values'].append(identifier)
identifiers.append(identifier)
cache[current_path] = config, identifier
break
else:
if option.isdynamic():
parent_option = self.config.option(path.rsplit('.', 1)[0])
identifiers = self._get_identifier(parent_option.name(uncalculated=True),
parent_option.name(),
)
dynamic_variable = None
while True:
dynamic_variable = parent_option.information.get('dynamic_variable',
None,
)
if dynamic_variable:
break
parent_option = self.config.option(parent_option.path().rsplit('.', 1)[0])
if '.' not in parent_option.path():
parent_option = None
break
if not parent_option:
continue
identifiers = parent_option.identifiers()
for identifier in identifiers:
dynamic_variable = dynamic_variable.replace('{{ identifier }}', str(identifier), 1)
if dynamic_variable not in values:
values[dynamic_variable] = {'values': []}
added.append(dynamic_variable)
elif dynamic_variable not in added:
continue
option_type = option.information.get('type')
typ = CONVERT_OPTION.get(option_type, {}).get("func")
if typ:
identifier = typ(identifier)
if identifier not in values[dynamic_variable]['values']:
values[dynamic_variable]['values'].append(identifier)
cache[option.path()] = option, identifier
def _get_identifier(self, true_name, name) -> str:
regexp = true_name.replace("{{ identifier }}", "(.*)")
finded = findall(regexp, name)
if len(finded) != 1 or not finded[0]:
return
return finded[0]
def convert_value(option, value):
if value == '':
return None
option_type = option.information.get('type')
func = CONVERT_OPTION.get(option_type, {}).get("func")
if func:
return func(value)
return value
__all__ = ("Rougail", "RougailConfig", "RougailUpgrade")

8
src/rougail/annotator/variable.py
View file

@ -178,13 +178,7 @@ class Annotator(Walk): # pylint: disable=R0903
msg = _(f'the variable "{variable.path}" has regexp attribut but has not the "regexp" type')
raise DictConsistencyError(msg, 37, variable.xmlfiles)
if variable.mandatory is None:
family_path = variable.path
hidden = variable.hidden is True
while '.' in family_path and not hidden:
family_path = family_path.rsplit(".", 1)[0]
family = self.objectspace.paths[family_path]
hidden = family.hidden is True
variable.mandatory = not hidden
variable.mandatory = True
def convert_test(self):
"""Convert variable tests value"""

35
src/rougail/config.py
View file

@ -193,7 +193,7 @@ def get_rougail_config(*,
main_namespace_default = 'rougail'
else:
main_namespace_default = 'null'
rougail_options = """default_dictionary_format_version:
rougail_options = f"""default_dictionary_format_version:
description: Dictionary format version by default, if not specified in dictionary file
alternative_name: v
choices:
@ -219,7 +219,7 @@ sort_dictionaries_all:
main_namespace:
description: Main namespace name
default: MAIN_MAMESPACE_DEFAULT
default: {main_namespace_default}
alternative_name: s
mandatory: false
@ -289,18 +289,29 @@ functions_files:
modes_level:
description: All modes level available
multi: true
mandatory: false
"""
if backward_compatibility:
rougail_options += """
default:
- basic
- standard
- advanced
commandline: false
"""
rougail_options += """
default_family_mode:
description: Default mode for a family
default:
type: jinja
jinja: |
{% if modes_level %}
{{ modes_level[0] }}
{% endif %}
disabled:
jinja: |
{% if not modes_level %}
No mode
{% endif %}
validators:
- type: jinja
jinja: |
@ -312,9 +323,19 @@ default_family_mode:
default_variable_mode:
description: Default mode for a variable
default:
type: jinja
jinja: |
{% if modes_level %}
{% if modes_level | length == 1 %}
{{ modes_level[0] }}
{% else %}
{{ modes_level[1] }}
{% endif %}
{% endif %}
disabled:
jinja: |
{% if not modes_level %}
No mode
{% endif %}
validators:
- type: jinja
jinja: |
@ -364,7 +385,7 @@ suffix:
default: ''
mandatory: false
commandline: false
""".replace('MAIN_MAMESPACE_DEFAULT', main_namespace_default)
"""
processes = {'structural': [],
'output': [],
'user data': [],

116
src/rougail/convert.py
View file

@ -133,7 +133,7 @@ class Paths:
if not force and is_dynamic:
self._dynamics[path] = dynamic
def get_relative_path(self,
def get_full_path(self,
path: str,
current_path: str,
):
@ -148,21 +148,22 @@ class Paths:
def get_with_dynamic(
self,
path: str,
suffix_path: str,
identifier_path: str,
current_path: str,
version: str,
namespace: str,
xmlfiles: List[str],
) -> Any:
suffix = None
identifier = None
if version != '1.0' and self.regexp_relative.search(path):
path = self.get_relative_path(path,
current_path,
)
path = self.get_full_path(path,
current_path,
)
else:
path = get_realpath(path, suffix_path)
path = get_realpath(path, identifier_path)
dynamic = None
if not path in self._data and "{{ suffix }}" not in path:
# version 1.0
if version == "1.0" and not path in self._data and "{{ identifier }}" not in path:
new_path = None
current_path = None
for name in path.split("."):
@ -184,10 +185,9 @@ class Paths:
parent_dynamic = None
name_dynamic = dynamic_path
if (
version == "1.0"
and parent_dynamic == parent_path
and name_dynamic.endswith("{{ suffix }}")
and name == name_dynamic.replace("{{ suffix }}", "")
parent_dynamic == parent_path
and name_dynamic.endswith("{{ identifier }}")
and name == name_dynamic.replace("{{ identifier }}", "")
):
new_path += "." + name_dynamic
break
@ -197,12 +197,12 @@ class Paths:
else:
new_path = name
path = new_path
if not path in self._data:
if version != "1.0" and not path in self._data:
current_path = None
parent_path = None
new_path = current_path
suffixes = []
identifiers = []
for name in path.split("."):
parent_path = current_path
if current_path:
current_path += "." + name
else:
@ -213,6 +213,7 @@ class Paths:
new_path += "." + name
else:
new_path = name
parent_path = current_path
continue
for dynamic_path in self._dynamics:
if '.' in dynamic_path:
@ -221,30 +222,35 @@ class Paths:
parent_dynamic = None
name_dynamic = dynamic_path
if (
"{{ suffix }}" not in name_dynamic
"{{ identifier }}" not in name_dynamic
or parent_path != parent_dynamic
):
continue
regexp = "^" + name_dynamic.replace("{{ suffix }}", "(.*)")
regexp = "^" + name_dynamic.replace("{{ identifier }}", "(.*)")
finded = findall(regexp, name)
if len(finded) != 1 or not finded[0]:
continue
suffixes.append(finded[0])
if finded[0] == "{{ identifier }}":
identifiers.append(None)
else:
identifiers.append(finded[0])
if new_path is None:
new_path = name_dynamic
else:
new_path += "." + name_dynamic
parent_path = dynamic_path
break
else:
if new_path:
new_path += "." + name
else:
new_path = name
if "{{ suffix }}" in name:
suffixes.append(None)
if "{{ identifier }}" in name:
identifiers.append(None)
parent_path = current_path
path = new_path
else:
suffixes = None
identifiers = None
if path not in self._data:
return None, None
option = self._data[path]
@ -258,7 +264,7 @@ class Paths:
f'shall not be used in the "{namespace}" namespace'
)
raise DictConsistencyError(msg, 38, xmlfiles)
return option, suffixes
return option, identifiers
def __getitem__(
self,
@ -316,8 +322,8 @@ class Informations:
class ParserVariable:
def __init__(self, rougailconfig):
self.load_config(rougailconfig)
self.rougailconfig = rougailconfig
self.load_config()
self.paths = Paths(self.main_namespace)
self.families = []
self.variables = []
@ -342,12 +348,14 @@ class ParserVariable:
self.is_init = False
super().__init__()
def load_config(self,
rougailconfig: 'RougailConfig',
) -> None:
self.rougailconfig = rougailconfig
def load_config(self) -> None:
rougailconfig = self.rougailconfig
self.sort_dictionaries_all = rougailconfig['sort_dictionaries_all']
try:
self.main_dictionaries = rougailconfig["main_dictionaries"]
except:
self.main_dictionaries = []
self.main_namespace = rougailconfig["main_namespace"]
self.main_dictionaries = rougailconfig["main_dictionaries"]
if self.main_namespace:
self.extra_dictionaries = rougailconfig["extra_dictionaries"]
self.suffix = rougailconfig["suffix"]
@ -355,14 +363,15 @@ class ParserVariable:
self.custom_types = rougailconfig["custom_types"]
self.functions_files = rougailconfig["functions_files"]
self.modes_level = rougailconfig["modes_level"]
self.default_variable_mode = rougailconfig["default_variable_mode"]
self.default_family_mode = rougailconfig["default_family_mode"]
if self.modes_level:
self.default_variable_mode = rougailconfig["default_variable_mode"]
self.default_family_mode = rougailconfig["default_family_mode"]
self.extra_annotators = rougailconfig["extra_annotators"]
self.base_option_name = rougailconfig["base_option_name"]
self.export_with_import = rougailconfig["export_with_import"]
self.internal_functions = rougailconfig["internal_functions"]
self.add_extra_options = rougailconfig["structural_commandline.add_extra_options"]
self.plugins = []
self.plugins = rougailconfig["plugins"]
def _init(self):
if self.is_init:
@ -557,8 +566,10 @@ class ParserVariable:
# it's just for modify subfamily or subvariable, do not redefine
if family_obj:
if not obj.pop("redefine", False):
raise Exception(
f"The family {path} already exists and she is not redefined in {filename}"
raise DictConsistencyError(
f'The family "{path}" already exists and it is not redefined',
32,
[filename],
)
# convert to Calculation objects
self.parse_parameters(
@ -604,12 +615,12 @@ class ParserVariable:
if obj_type == "dynamic":
family_is_dynamic = True
parent_dynamic = path
if '{{ suffix }}' not in name:
if '{{ identifier }}' not in name:
if "variable" in family_obj:
name += '{{ suffix }}'
path += '{{ suffix }}'
name += '{{ identifier }}'
path += '{{ identifier }}'
else:
msg = f'dynamic family name must have "{{{{ suffix }}}}" in his name for "{path}"'
msg = f'dynamic family name must have "{{{{ identifier }}}}" in his name for "{path}"'
raise DictConsistencyError(msg, 13, [filename])
if version != '1.0' and not family_obj and comment:
family_obj['description'] = comment
@ -938,6 +949,7 @@ class ParserVariable:
variable["namespace"] = self.namespace
variable["version"] = version
variable["path_prefix"] = self.path_prefix
variable["xmlfiles"] = filename
variable_type = self.get_family_or_variable_type(variable)
obj = {
@ -1074,8 +1086,8 @@ class ParserVariable:
try:
params.append(PARAM_TYPES[param_typ](**val))
except ValidationError as err:
raise Exception(
f'"{attribute}" has an invalid "{key}" for {path}: {err}'
raise DictConsistencyError(
f'"{attribute}" has an invalid "{key}" for {path}: {err}', 29, xmlfiles
) from err
calculation_object["params"] = params
#
@ -1086,8 +1098,8 @@ class ParserVariable:
f'unknown "return_type" in {attribute} of variable "{path}"'
)
#
if typ == "suffix" and not family_is_dynamic:
msg = f'suffix calculation for "{attribute}" in "{path}" cannot be set none dynamic family'
if typ == "identifier" and not family_is_dynamic:
msg = f'identifier calculation for "{attribute}" in "{path}" cannot be set none dynamic family'
raise DictConsistencyError(msg, 53, xmlfiles)
if attribute in PROPERTY_ATTRIBUTE:
calc = CALCULATION_PROPERTY_TYPES[typ](**calculation_object)
@ -1143,15 +1155,16 @@ class RougailConvert(ParserVariable):
"""Parse directories content"""
self._init()
if path_prefix:
if path_prefix in self.parents:
n_path_prefix = normalize_family(path_prefix)
if n_path_prefix in self.parents:
raise Exception("pfffff")
root_parent = path_prefix
self.path_prefix = path_prefix
root_parent = n_path_prefix
self.path_prefix = n_path_prefix
self.namespace = None
self.add_family(
path_prefix,
path_prefix,
{},
n_path_prefix,
n_path_prefix,
{'description': path_prefix},
"",
False,
None,
@ -1267,11 +1280,14 @@ class RougailConvert(ParserVariable):
"""Sort filename"""
if not isinstance(directories, list):
directories = [directories]
if self.sort_dictionaries_all:
filenames = {}
for directory_name in directories:
directory = Path(directory_name)
if not directory.is_dir():
continue
filenames = {}
if not self.sort_dictionaries_all:
filenames = {}
for file_path in directory.iterdir():
if file_path.suffix not in [".yml", ".yaml"]:
continue
@ -1282,6 +1298,10 @@ class RougailConvert(ParserVariable):
[filenames[file_path.name][1]],
)
filenames[file_path.name] = str(file_path)
if not self.sort_dictionaries_all:
for filename in sorted(filenames):
yield filenames[filename]
if self.sort_dictionaries_all:
for filename in sorted(filenames):
yield filenames[filename]