stove/rougail
3
0
Fork 0
Code Issues 10 Pull requests 1 Projects Releases Wiki Activity

WIP: Expand the developer documentation #27

Draft
gremond wants to merge 62 commits from develop into developer_docs
pull from: develop
merge into: stove:developer_docs
Conversation 0 Commits 62 Files changed 4320 +649 -422
20 changed files with 649 additions and 422 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 22348f71e6 - Show all commits

146
src/rougail/__init__.py
View file

@ -27,6 +27,7 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from tiramisu import Config, undefined
from tiramisu.error import PropertiesOptionError, LeadershipError, ConfigError
from warnings import warn
@ -40,18 +41,21 @@ from .object_model import CONVERT_OPTION
from .utils import normalize_family
def tiramisu_display_name(kls,
def tiramisu_display_name(
kls,
subconfig,
with_quote: bool=False,
) -> str:
with_quote: bool = False,
) -> str:
"""Replace the Tiramisu display_name function to display path + description"""
doc = kls._get_information(subconfig, "doc", None)
comment = f" ({doc})" if doc and doc != kls.impl_getname() else ""
if "{{ identifier }}" in comment:
comment = comment.replace('{{ identifier }}', str(subconfig.identifiers[-1]))
comment = comment.replace("{{ identifier }}", str(subconfig.identifiers[-1]))
path = kls.impl_getpath()
if "{{ identifier }}" in path and subconfig.identifiers:
path = path.replace('{{ identifier }}', normalize_family(str(subconfig.identifiers[-1])))
path = path.replace(
"{{ identifier }}", normalize_family(str(subconfig.identifiers[-1]))
)
if with_quote:
return f'"{path}"{comment}'
return f"{path}{comment}"
@ -83,7 +87,10 @@ class Rougail:
if not self.config:
tiram_obj = self.converted.save(self.rougailconfig["tiramisu_cache"])
optiondescription = {}
custom_types = {custom.__name__: custom for custom in self.rougailconfig["custom_types"].values()}
custom_types = {
custom.__name__: custom
for custom in self.rougailconfig["custom_types"].values()
}
exec(tiram_obj, custom_types, optiondescription) # pylint: disable=W0122
self.config = Config(
optiondescription["option_0"],
@ -93,22 +100,26 @@ class Rougail:
return self.config
def get_config(self):
warn("get_config is deprecated, use run instead", DeprecationWarning, stacklevel=2)
warn(
"get_config is deprecated, use run instead",
DeprecationWarning,
stacklevel=2,
)
return self.run()
def user_datas(self,
user_datas: List[dict]):
def user_datas(self, user_datas: List[dict]):
values = {}
errors = []
warnings = []
for datas in user_datas:
options = datas.get('options', {})
for name, data in datas.get('values', {}).items():
values[name] = {'values': data,
'options': options.copy(),
options = datas.get("options", {})
for name, data in datas.get("values", {}).items():
values[name] = {
"values": data,
"options": options.copy(),
}
errors.extend(datas.get('errors', []))
warnings.extend(datas.get('warnings', []))
errors.extend(datas.get("errors", []))
warnings.extend(datas.get("warnings", []))
self._auto_configure_dynamics(values)
while values:
value_is_set = False
@ -116,21 +127,21 @@ class Rougail:
path = option.path()
if path not in values:
path = path.upper()
options = values.get(path, {}).get('options', {})
if path not in values or options.get('upper') is not True:
options = values.get(path, {}).get("options", {})
if path not in values or options.get("upper") is not True:
continue
else:
options = values[path].get('options', {})
options = values[path].get("options", {})
value = values[path]["values"]
if option.ismulti():
if options.get('multi_separator') and not isinstance(value, list):
value = value.split(options['multi_separator'])
if options.get("multi_separator") and not isinstance(value, list):
value = value.split(options["multi_separator"])
values[path]["values"] = value
if options.get('needs_convert'):
if options.get("needs_convert"):
value = [convert_value(option, val) for val in value]
values[path]["values"] = value
values[path]["options"]["needs_convert"] = False
elif options.get('needs_convert'):
elif options.get("needs_convert"):
value = convert_value(option, value)
index = option.index()
if index is not None:
@ -141,8 +152,8 @@ class Rougail:
option.value.set(value)
value_is_set = True
if index is not None:
values[path]['values'][index] = undefined
if set(values[path]['values']) == {undefined}:
values[path]["values"][index] = undefined
if set(values[path]["values"]) == {undefined}:
values.pop(path)
else:
values.pop(path)
@ -154,7 +165,7 @@ class Rougail:
for path, data in values.items():
try:
option = self.config.option(path)
value = data['values']
value = data["values"]
if option.isfollower():
for index, val in enumerate(value):
if val is undefined:
@ -165,13 +176,14 @@ class Rougail:
except AttributeError as err:
errors.append(str(err))
except (ValueError, LeadershipError) as err:
#errors.append(str(err).replace('"', "'"))
# errors.append(str(err).replace('"', "'"))
errors.append(str(err))
except PropertiesOptionError as err:
# warnings.append(f'"{err}" but is defined in "{self.filename}"')
# warnings.append(f'"{err}" but is defined in "{self.filename}"')
warnings.append(str(err))
return {'errors': errors,
'warnings': warnings,
return {
"errors": errors,
"warnings": warnings,
}
def _get_variable(self, config):
@ -181,14 +193,15 @@ class Rougail:
else:
yield subconfig
def _auto_configure_dynamics(self,
def _auto_configure_dynamics(
self,
values,
):
cache = {}
added = []
for path, data in list(values.items()):
value = data['values']
# for value in data['values'].items():
value = data["values"]
# for value in data['values'].items():
try:
option = self.config.option(path)
option.name()
@ -196,11 +209,11 @@ class Rougail:
pass
except AttributeError:
config = self.config
current_path = ''
current_path = ""
identifiers = []
for name in path.split('.')[:-1]:
for name in path.split(".")[:-1]:
if current_path:
current_path += '.'
current_path += "."
current_path += name
if current_path in cache:
config, identifier = cache[current_path]
@ -213,66 +226,86 @@ class Rougail:
except AttributeError:
for tconfig in config.list(uncalculated=True):
if tconfig.isdynamic(only_self=True):
identifier = self._get_identifier(tconfig.name(), name)
identifier = self._get_identifier(
tconfig.name(), name
)
if identifier is None:
continue
dynamic_variable = tconfig.information.get('dynamic_variable',
dynamic_variable = tconfig.information.get(
"dynamic_variable",
None,
)
if not dynamic_variable:
continue
option_type = self.config.option(dynamic_variable).information.get('type')
option_type = self.config.option(
dynamic_variable
).information.get("type")
if identifiers:
for s in identifiers:
dynamic_variable = dynamic_variable.replace('{{ identifier }}', str(s), 1)
dynamic_variable = dynamic_variable.replace(
"{{ identifier }}", str(s), 1
)
if dynamic_variable not in values:
values[dynamic_variable] = {'values': []}
values[dynamic_variable] = {"values": []}
added.append(dynamic_variable)
elif dynamic_variable not in added:
continue
config = tconfig
# option_type = option.information.get('type')
typ = CONVERT_OPTION.get(option_type, {}).get("func")
# option_type = option.information.get('type')
typ = CONVERT_OPTION.get(option_type, {}).get(
"func"
)
if typ:
identifier = typ(identifier)
if identifier not in values[dynamic_variable]['values']:
values[dynamic_variable]['values'].append(identifier)
if (
identifier
not in values[dynamic_variable]["values"]
):
values[dynamic_variable]["values"].append(
identifier
)
identifiers.append(identifier)
cache[current_path] = config, identifier
break
else:
if option.isdynamic():
parent_option = self.config.option(path.rsplit('.', 1)[0])
identifiers = self._get_identifier(parent_option.name(uncalculated=True),
parent_option = self.config.option(path.rsplit(".", 1)[0])
identifiers = self._get_identifier(
parent_option.name(uncalculated=True),
parent_option.name(),
)
dynamic_variable = None
while True:
dynamic_variable = parent_option.information.get('dynamic_variable',
dynamic_variable = parent_option.information.get(
"dynamic_variable",
None,
)
if dynamic_variable:
break
parent_option = self.config.option(parent_option.path().rsplit('.', 1)[0])
if '.' not in parent_option.path():
parent_option = self.config.option(
parent_option.path().rsplit(".", 1)[0]
)
if "." not in parent_option.path():
parent_option = None
break
if not parent_option:
continue
identifiers = parent_option.identifiers()
for identifier in identifiers:
dynamic_variable = dynamic_variable.replace('{{ identifier }}', str(identifier), 1)
dynamic_variable = dynamic_variable.replace(
"{{ identifier }}", str(identifier), 1
)
if dynamic_variable not in values:
values[dynamic_variable] = {'values': []}
values[dynamic_variable] = {"values": []}
added.append(dynamic_variable)
elif dynamic_variable not in added:
continue
option_type = option.information.get('type')
option_type = option.information.get("type")
typ = CONVERT_OPTION.get(option_type, {}).get("func")
if typ:
identifier = typ(identifier)
if identifier not in values[dynamic_variable]['values']:
values[dynamic_variable]['values'].append(identifier)
if identifier not in values[dynamic_variable]["values"]:
values[dynamic_variable]["values"].append(identifier)
cache[option.path()] = option, identifier
def _get_identifier(self, true_name, name) -> str:
@ -282,10 +315,11 @@ class Rougail:
return
return finded[0]
def convert_value(option, value):
if value == '':
if value == "":
return None
option_type = option.information.get('type')
option_type = option.information.get("type")
func = CONVERT_OPTION.get(option_type, {}).get("func")
if func:
return func(value)

7
src/rougail/annotator/__init__.py
View file

@ -27,6 +27,7 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
import importlib.resources
from os.path import isfile
from ..utils import load_modules
@ -68,20 +69,20 @@ class SpaceAnnotator: # pylint: disable=R0903
get_annotators(ANNOTATORS, extra_annotator)
for plugin in objectspace.plugins:
try:
get_annotators(ANNOTATORS, f'rougail.{plugin}.annotator')
get_annotators(ANNOTATORS, f"rougail.{plugin}.annotator")
except ModuleNotFoundError:
pass
annotators = ANNOTATORS["rougail.annotator"].copy()
for extra_annotator in objectspace.extra_annotators:
annotators.extend(ANNOTATORS[extra_annotator])
for plugin in objectspace.plugins:
annotators.extend(ANNOTATORS[f'rougail.{plugin}.annotator'])
annotators.extend(ANNOTATORS[f"rougail.{plugin}.annotator"])
annotators = sorted(annotators, key=get_level)
functions = {}
functions_files = objectspace.functions_files
for functions_file in functions_files:
if isfile(functions_file):
loaded_modules = load_modules('function_file', functions_file)
loaded_modules = load_modules("function_file", functions_file)
for function in dir(loaded_modules):
if function.startswith("_"):
continue

44
src/rougail/annotator/family.py
View file

@ -27,6 +27,7 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from typing import Optional
from rougail.i18n import _
from rougail.error import DictConsistencyError
@ -69,8 +70,7 @@ class Annotator(Walk):
self.family_description()
if self.objectspace.modes_level:
self.modes = {
name: Mode(idx)
for idx, name in enumerate(self.objectspace.modes_level)
name: Mode(idx) for idx, name in enumerate(self.objectspace.modes_level)
}
self.default_variable_mode = self.objectspace.default_variable_mode
self.default_family_mode = self.objectspace.default_family_mode
@ -100,7 +100,10 @@ class Annotator(Walk):
if isinstance(family, self.objectspace.family) and not self._has_variable(
family.path
):
if self.objectspace.paths.default_namespace is None or "." in family.path:
if (
self.objectspace.paths.default_namespace is None
or "." in family.path
):
removed_families.append(family.path)
removed_families.reverse()
for family in removed_families:
@ -122,8 +125,11 @@ class Annotator(Walk):
for family in self.get_families():
if not family.description:
family.description = family.name
if family.type == 'dynamic' and isinstance(family.dynamic, VariableCalculation):
path = self.objectspace.paths.get_full_path(family.dynamic.variable,
if family.type == "dynamic" and isinstance(
family.dynamic, VariableCalculation
):
path = self.objectspace.paths.get_full_path(
family.dynamic.variable,
family.path,
)
self.objectspace.informations.add(family.path, "dynamic_variable", path)
@ -153,11 +159,15 @@ class Annotator(Walk):
for family in families:
self._change_family_mode(family)
if self.objectspace.paths.default_namespace is None:
for variable_path in self.objectspace.parents['.']:
for variable_path in self.objectspace.parents["."]:
variable = self.objectspace.paths[variable_path]
if variable.type == "symlink" or variable_path in self.objectspace.families:
if (
variable.type == "symlink"
or variable_path in self.objectspace.families
):
continue
self._set_default_mode_variable(variable,
self._set_default_mode_variable(
variable,
self.default_variable_mode,
check_level=False,
)
@ -220,7 +230,7 @@ class Annotator(Walk):
self,
variable: "self.objectspace.variable",
family_mode: Optional[str],
check_level: bool=True,
check_level: bool = True,
) -> None:
# auto_save variable is set to 'basic' mode
# if its mode is not defined by the user
@ -234,7 +244,11 @@ class Annotator(Walk):
and variable.path not in self.objectspace.default_multi
):
variable_mode = self.objectspace.modes_level[0]
if check_level and family_mode and self.modes[variable_mode] < self.modes[family_mode]:
if (
check_level
and family_mode
and self.modes[variable_mode] < self.modes[family_mode]
):
msg = _(
f'the variable "{variable.name}" is mandatory so in "{variable_mode}" mode '
f'but family has the higher family mode "{family_mode}"'
@ -261,9 +275,7 @@ class Annotator(Walk):
if leader == follower:
# it's a leader
if not leader.mode:
self._set_auto_mode(
leader, self.default_variable_mode
)
self._set_auto_mode(leader, self.default_variable_mode)
return
if self._has_mode(follower):
follower_mode = follower.mode
@ -310,8 +322,10 @@ class Annotator(Walk):
# set the lower variable mode to family
self._set_auto_mode(family, min_variable_mode)
if self.modes[family.mode] < self.modes[min_variable_mode]:
msg = _(f'the family "{family.name}" is in "{family.mode}" mode but variables and '
f'families inside have the higher modes "{min_variable_mode}"')
msg = _(
f'the family "{family.name}" is in "{family.mode}" mode but variables and '
f'families inside have the higher modes "{min_variable_mode}"'
)
raise DictConsistencyError(msg, 62, family.xmlfiles)
def _change_variable_mode(

3
src/rougail/annotator/property.py
View file

@ -27,6 +27,7 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from typing import Union
from rougail.i18n import _
from rougail.error import DictConsistencyError
@ -123,7 +124,7 @@ class Annotator(Walk):
value = []
for calculation in frozen:
calculation_copy = calculation.copy()
calculation_copy.attribute_name = 'frozen'
calculation_copy.attribute_name = "frozen"
calculation_copy.ori_path = calculation_copy.path
calculation_copy.path = path
value.append(calculation_copy)

17
src/rougail/annotator/value.py
View file

@ -27,6 +27,7 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from rougail.annotator.variable import Walk
from rougail.i18n import _
@ -55,7 +56,7 @@ class Annotator(Walk): # pylint: disable=R0903
for variable in self.get_variables():
if variable.type == "symlink":
continue
if variable.version != '1.0' and variable.type == 'port':
if variable.version != "1.0" and variable.type == "port":
self._convert_port(variable)
self._convert_value(variable)
@ -85,16 +86,16 @@ class Annotator(Walk): # pylint: disable=R0903
else:
if variable.path not in self.objectspace.leaders:
if multi == "submulti":
self.objectspace.default_multi[
variable.path
] = variable.default
self.objectspace.default_multi[variable.path] = variable.default
variable.default = None
else:
self.objectspace.default_multi[variable.path] = variable.default[
0
]
self.objectspace.default_multi[variable.path] = (
variable.default[0]
)
elif variable.multi:
msg = _(f'the variable "{variable.name}" is multi but has a non list default value')
msg = _(
f'the variable "{variable.name}" is multi but has a non list default value'
)
raise DictConsistencyError(msg, 12, variable.xmlfiles)
elif variable.path in self.objectspace.followers:
self.objectspace.default_multi[variable.path] = variable.default

58
src/rougail/annotator/variable.py
View file

@ -66,15 +66,18 @@ class Annotator(Walk): # pylint: disable=R0903
return
self.objectspace = objectspace
if self.objectspace.main_namespace:
self.forbidden_name = [
self.objectspace.main_namespace
]
self.forbidden_name = [self.objectspace.main_namespace]
for extra in self.objectspace.extra_dictionaries:
self.forbidden_name.append(extra)
else:
self.forbidden_name = []
# default type inference from a default value with :term:`basic types`
self.basic_types = {str: "string", int: "number", bool: "boolean", float: "float"}
self.basic_types = {
str: "string",
int: "number",
bool: "boolean",
float: "float",
}
self.verify_choices()
self.convert_variable()
self.convert_test()
@ -97,9 +100,7 @@ class Annotator(Walk): # pylint: disable=R0903
variable.multi = False
if variable.type is None:
variable.type = "string"
self.objectspace.informations.add(
variable.path, "type", variable.type
)
self.objectspace.informations.add(variable.path, "type", variable.type)
self._convert_variable(variable)
def _convert_variable_inference(
@ -120,7 +121,9 @@ class Annotator(Walk): # pylint: disable=R0903
tested_value = variable.default
variable.type = self.basic_types.get(type(tested_value), None)
# variable has no multi attribute
if variable.multi is None and not (variable.type is None and isinstance(variable.default, VariableCalculation)):
if variable.multi is None and not (
variable.type is None and isinstance(variable.default, VariableCalculation)
):
if variable.path in self.objectspace.leaders:
variable.multi = True
else:
@ -131,12 +134,19 @@ class Annotator(Walk): # pylint: disable=R0903
variable,
) -> None:
# if a variable has a variable as default value, that means the type/params or multi should has same value
if variable.type is not None or not isinstance(variable.default, VariableCalculation):
if variable.type is not None or not isinstance(
variable.default, VariableCalculation
):
return
# copy type and params
calculated_variable_path = variable.default.variable
calculated_variable, identifier = self.objectspace.paths.get_with_dynamic(
calculated_variable_path, variable.default.path_prefix, variable.path, variable.version, variable.namespace, variable.xmlfiles
calculated_variable_path,
variable.default.path_prefix,
variable.path,
variable.version,
variable.namespace,
variable.xmlfiles,
)
if calculated_variable is None:
return
@ -146,7 +156,11 @@ class Annotator(Walk): # pylint: disable=R0903
# copy multi attribut
if variable.multi is None:
calculated_path = calculated_variable.path
if calculated_path in self.objectspace.leaders and variable.path in self.objectspace.followers and calculated_path.rsplit('.')[0] == variable.path.rsplit('.')[0]:
if (
calculated_path in self.objectspace.leaders
and variable.path in self.objectspace.followers
and calculated_path.rsplit(".")[0] == variable.path.rsplit(".")[0]
):
variable.multi = False
else:
variable.multi = calculated_variable.multi
@ -171,11 +185,13 @@ class Annotator(Walk): # pylint: disable=R0903
family = self.objectspace.paths[variable.path.rsplit(".", 1)[0]]
if variable.hidden:
family.hidden = variable.hidden
# elif family.hidden:
# variable.hidden = family.hidden
# elif family.hidden:
# variable.hidden = family.hidden
variable.hidden = None
if variable.regexp is not None and variable.type != 'regexp':
msg = _(f'the variable "{variable.path}" has regexp attribut but has not the "regexp" type')
if variable.regexp is not None and variable.type != "regexp":
msg = _(
f'the variable "{variable.path}" has regexp attribut but has not the "regexp" type'
)
raise DictConsistencyError(msg, 37, variable.xmlfiles)
if variable.mandatory is None:
variable.mandatory = True
@ -215,10 +231,12 @@ class Annotator(Walk): # pylint: disable=R0903
if variable.type is None and variable.choices:
# choice type inference from the `choices` attribute
variable.type = "choice"
if variable.choices is not None and variable.type != 'choice':
msg = _(f'the variable "{variable.path}" has choices attribut but has not the "choice" type')
if variable.choices is not None and variable.type != "choice":
msg = _(
f'the variable "{variable.path}" has choices attribut but has not the "choice" type'
)
raise DictConsistencyError(msg, 11, variable.xmlfiles)
if variable.type != 'choice':
if variable.type != "choice":
continue
if variable.default is None:
continue
@ -242,5 +260,7 @@ class Annotator(Walk): # pylint: disable=R0903
if isinstance(value, Calculation):
continue
if value not in choices:
msg = _(f'the variable "{variable.path}" has an unvalid default value "{value}" should be in {display_list(choices, separator="or", add_quote=True)}')
msg = _(
f'the variable "{variable.path}" has an unvalid default value "{value}" should be in {display_list(choices, separator="or", add_quote=True)}'
)
raise DictConsistencyError(msg, 26, variable.xmlfiles)

133
src/rougail/config.py
View file

@ -28,6 +28,7 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from pathlib import Path
from tiramisu import Config
from ruamel.yaml import YAML
@ -35,12 +36,14 @@ from .utils import _, load_modules, normalize_family
from .convert import RougailConvert
RENAMED = {'dictionaries_dir': 'main_dictionaries',
'variable_namespace': 'main_namespace',
'functions_file': 'functions_files',
}
NOT_IN_TIRAMISU = {'custom_types': {},
}
RENAMED = {
"dictionaries_dir": "main_dictionaries",
"variable_namespace": "main_namespace",
"functions_file": "functions_files",
}
NOT_IN_TIRAMISU = {
"custom_types": {},
}
SUBMODULES = None
@ -49,24 +52,22 @@ def get_sub_modules():
if SUBMODULES is None:
SUBMODULES = {}
for submodule in Path(__file__).parent.iterdir():
if submodule.name.startswith('_') or not submodule.is_dir():
if submodule.name.startswith("_") or not submodule.is_dir():
continue
config_file = submodule / 'config.py'
config_file = submodule / "config.py"
if config_file.is_file():
SUBMODULES[submodule.name] = load_modules('rougail.' + submodule.name + '.config', str(config_file))
SUBMODULES[submodule.name] = load_modules(
"rougail." + submodule.name + ".config", str(config_file)
)
return SUBMODULES
def get_level(module):
return module['level']
return module["level"]
class _RougailConfig:
def __init__(self,
backward_compatibility: bool,
root,
extra_vars: dict
):
def __init__(self, backward_compatibility: bool, root, extra_vars: dict):
self.backward_compatibility = backward_compatibility
self.root = root
self.config = Config(
@ -81,7 +82,9 @@ class _RougailConfig:
setattr(self, variable, default_value)
def copy(self):
rougailconfig = _RougailConfig(self.backward_compatibility, self.root, self.extra_vars)
rougailconfig = _RougailConfig(
self.backward_compatibility, self.root, self.extra_vars
)
rougailconfig.config.value.importation(self.config.value.exportation())
rougailconfig.config.property.importation(self.config.property.exportation())
rougailconfig.config.property.read_only()
@ -92,7 +95,8 @@ class _RougailConfig:
setattr(rougailconfig, variable, value)
return rougailconfig
def __setitem__(self,
def __setitem__(
self,
key,
value,
) -> None:
@ -100,8 +104,8 @@ class _RougailConfig:
setattr(self, key, value)
else:
self.config.property.read_write()
if key == 'export_with_import':
key = 'not_export_with_import'
if key == "export_with_import":
key = "not_export_with_import"
key = RENAMED.get(key, key)
option = self.config.option(key)
if option.isoptiondescription() and option.isleadership():
@ -111,30 +115,29 @@ class _RougailConfig:
follower = option.followers()[0]
for idx, val in enumerate(value.values()):
self.config.option(follower.path(), idx).value.set(val)
elif key == 'not_export_with_import':
elif key == "not_export_with_import":
option.value.set(not value)
else:
option.value.set(value)
self.config.property.read_only()
def __getitem__(self,
def __getitem__(
self,
key,
) -> None:
if key in self.not_in_tiramisu:
return getattr(self, key)
if key == 'export_with_import':
key = 'not_export_with_import'
if key == "export_with_import":
key = "not_export_with_import"
option = self.config.option(key)
if option.isoptiondescription() and option.isleadership():
return self.get_leadership(option)
ret = self.config.option(key).value.get()
if key == 'not_export_with_import':
if key == "not_export_with_import":
return not ret
return ret
def get_leadership(self,
option
) -> dict:
def get_leadership(self, option) -> dict:
leader = None
followers = []
for opt, value in option.value.get().items():
@ -151,7 +154,7 @@ class _RougailConfig:
if option.isoptiondescription():
yield from self.parse(option)
elif not option.issymlinkoption():
yield f'{option.path()}: {option.value.get()}'
yield f"{option.path()}: {option.value.get()}"
def __repr__(self):
self.config.property.read_write()
@ -164,7 +167,8 @@ class _RougailConfig:
class FakeRougailConvert(RougailConvert):
def __init__(self,
def __init__(
self,
add_extra_options: bool,
) -> None:
self.add_extra_options = add_extra_options
@ -173,7 +177,7 @@ class FakeRougailConvert(RougailConvert):
def load_config(self) -> None:
self.sort_dictionaries_all = False
self.main_namespace = None
self.suffix = ''
self.suffix = ""
self.custom_types = {}
self.functions_files = []
self.modes_level = []
@ -181,18 +185,19 @@ class FakeRougailConvert(RougailConvert):
self.base_option_name = "baseoption"
self.export_with_import = True
self.internal_functions = []
self.plugins = ['structural_commandline']
self.plugins = ["structural_commandline"]
self.add_extra_options = self.add_extra_options
def get_rougail_config(*,
backward_compatibility: bool=True,
add_extra_options: bool=True,
) -> _RougailConfig:
def get_rougail_config(
*,
backward_compatibility: bool = True,
add_extra_options: bool = True,
) -> _RougailConfig:
if backward_compatibility:
main_namespace_default = 'rougail'
main_namespace_default = "rougail"
else:
main_namespace_default = 'null'
main_namespace_default = "null"
rougail_options = f"""default_dictionary_format_version:
description: Dictionary format version by default, if not specified in dictionary file
alternative_name: v
@ -386,13 +391,14 @@ suffix:
mandatory: false
commandline: false
"""
processes = {'structural': [],
'output': [],
'user data': [],
processes = {
"structural": [],
"output": [],
"user data": [],
}
for module in get_sub_modules().values():
data = module.get_rougail_config()
processes[data['process']].append(data)
processes[data["process"]].append(data)
# reorder
for process in processes:
processes[process] = list(sorted(processes[process], key=get_level))
@ -407,17 +413,22 @@ suffix:
description: Select for {NAME}
alternative_name: {NAME[0]}
choices:
""".format(NAME=normalize_family(process),
""".format(
NAME=normalize_family(process),
)
for obj in objects:
rougail_process += f" - {obj['name']}\n"
if process == 'structural':
if process == "structural":
rougail_process += " commandline: false"
elif process == 'user data':
elif process == "user data":
rougail_process += """ multi: true
mandatory: false
"""
hidden_outputs = [process['name'] for process in processes['output'] if not process.get('allow_user_data', True)]
hidden_outputs = [
process["name"]
for process in processes["output"]
if not process.get("allow_user_data", True)
]
if hidden_outputs:
rougail_process += """ hidden:
type: jinja
@ -427,9 +438,13 @@ suffix:
rougail_process += """ {% if _.output == 'NAME' %}
Cannot load user data for NAME output
{% endif %}
""".replace('NAME', hidden_output)
""".replace(
"NAME", hidden_output
)
else:
rougail_process += ' default: {DEFAULT}'.format(DEFAULT=objects[0]['name'])
rougail_process += " default: {DEFAULT}".format(
DEFAULT=objects[0]["name"]
)
else:
rougail_process += """
{NAME}:
@ -437,36 +452,38 @@ suffix:
hidden: true
mandatory: false
multi: true
""".format(NAME=normalize_family(process),
""".format(
NAME=normalize_family(process),
)
rougail_options += rougail_process
convert = FakeRougailConvert(add_extra_options)
convert._init()
convert.namespace = None
convert.parse_root_file(
'rougail.config',
'',
'1.1',
"rougail.config",
"",
"1.1",
YAML().load(rougail_options),
)
extra_vars = {}
for process in processes:
for obj in processes[process]:
if 'extra_vars' in obj:
extra_vars |= obj['extra_vars']
if not 'options' in obj:
if "extra_vars" in obj:
extra_vars |= obj["extra_vars"]
if not "options" in obj:
continue
convert.parse_root_file(
f'rougail.config.{obj["name"]}',
'',
'1.1',
YAML().load(obj['options']),
"",
"1.1",
YAML().load(obj["options"]),
)
tiram_obj = convert.save(None)
optiondescription = {}
exec(tiram_obj, {}, optiondescription) # pylint: disable=W0122
return _RougailConfig(backward_compatibility,
return _RougailConfig(
backward_compatibility,
optiondescription["option_0"],
extra_vars=extra_vars,
)

133
src/rougail/convert.py
View file

@ -28,6 +28,7 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
import logging
from itertools import chain
from pathlib import Path
@ -133,7 +134,8 @@ class Paths:
if not force and is_dynamic:
self._dynamics[path] = dynamic
def get_full_path(self,
def get_full_path(
self,
path: str,
current_path: str,
):
@ -155,15 +157,20 @@ class Paths:
xmlfiles: List[str],
) -> Any:
identifier = None
if version != '1.0' and self.regexp_relative.search(path):
path = self.get_full_path(path,
if version != "1.0" and self.regexp_relative.search(path):
path = self.get_full_path(
path,
current_path,
)
else:
path = get_realpath(path, identifier_path)
dynamic = None
# version 1.0
if version == "1.0" and not path in self._data and "{{ identifier }}" not in path:
if (
version == "1.0"
and not path in self._data
and "{{ identifier }}" not in path
):
new_path = None
current_path = None
for name in path.split("."):
@ -179,7 +186,7 @@ class Paths:
new_path = name
continue
for dynamic_path in self._dynamics:
if '.' in dynamic_path:
if "." in dynamic_path:
parent_dynamic, name_dynamic = dynamic_path.rsplit(".", 1)
else:
parent_dynamic = None
@ -216,7 +223,7 @@ class Paths:
parent_path = current_path
continue
for dynamic_path in self._dynamics:
if '.' in dynamic_path:
if "." in dynamic_path:
parent_dynamic, name_dynamic = dynamic_path.rsplit(".", 1)
else:
parent_dynamic = None
@ -350,7 +357,7 @@ class ParserVariable:
def load_config(self) -> None:
rougailconfig = self.rougailconfig
self.sort_dictionaries_all = rougailconfig['sort_dictionaries_all']
self.sort_dictionaries_all = rougailconfig["sort_dictionaries_all"]
try:
self.main_dictionaries = rougailconfig["main_dictionaries"]
except:
@ -359,7 +366,9 @@ class ParserVariable:
if self.main_namespace:
self.extra_dictionaries = rougailconfig["extra_dictionaries"]
self.suffix = rougailconfig["suffix"]
self.default_dictionary_format_version = rougailconfig["default_dictionary_format_version"]
self.default_dictionary_format_version = rougailconfig[
"default_dictionary_format_version"
]
self.custom_types = rougailconfig["custom_types"]
self.functions_files = rougailconfig["functions_files"]
self.modes_level = rougailconfig["modes_level"]
@ -370,7 +379,9 @@ class ParserVariable:
self.base_option_name = rougailconfig["base_option_name"]
self.export_with_import = rougailconfig["export_with_import"]
self.internal_functions = rougailconfig["internal_functions"]
self.add_extra_options = rougailconfig["structural_commandline.add_extra_options"]
self.add_extra_options = rougailconfig[
"structural_commandline.add_extra_options"
]
self.plugins = rougailconfig["plugins"]
def _init(self):
@ -381,14 +392,22 @@ class ParserVariable:
if self.plugins:
root = Path(__file__).parent
for plugin in self.plugins:
module_path = root / plugin / 'object_model.py'
module_path = root / plugin / "object_model.py"
if not module_path.is_file():
continue
module = load_modules(f'rougail.{plugin}.object_model', str(module_path))
if 'Variable' in module.__all__:
variable = type(variable.__name__ + '_' + plugin, (variable, module.Variable), {})
if 'Family' in module.__all__:
family = type(family.__name__ + '_' + plugin, (family, module.Family), {})
module = load_modules(
f"rougail.{plugin}.object_model", str(module_path)
)
if "Variable" in module.__all__:
variable = type(
variable.__name__ + "_" + plugin,
(variable, module.Variable),
{},
)
if "Family" in module.__all__:
family = type(
family.__name__ + "_" + plugin, (family, module.Family), {}
)
self.variable = variable
self.family = family
self.dynamic = type(Dynamic.__name__, (Dynamic, family), {})
@ -604,26 +623,26 @@ class ParserVariable:
obj_type = self.get_family_or_variable_type(family_obj)
if obj_type is None:
# auto set type
if '_dynamic' in family_obj:
dynamic = family_obj['_dynamic']
elif 'dynamic' in family_obj:
dynamic = family_obj['dynamic']
if "_dynamic" in family_obj:
dynamic = family_obj["_dynamic"]
elif "dynamic" in family_obj:
dynamic = family_obj["dynamic"]
else:
dynamic = None
if isinstance(dynamic, (list, dict)):
family_obj['type'] = obj_type = 'dynamic'
family_obj["type"] = obj_type = "dynamic"
if obj_type == "dynamic":
family_is_dynamic = True
parent_dynamic = path
if '{{ identifier }}' not in name:
if "{{ identifier }}" not in name:
if "variable" in family_obj:
name += '{{ identifier }}'
path += '{{ identifier }}'
name += "{{ identifier }}"
path += "{{ identifier }}"
else:
msg = f'dynamic family name must have "{{{{ identifier }}}}" in his name for "{path}"'
raise DictConsistencyError(msg, 13, [filename])
if version != '1.0' and not family_obj and comment:
family_obj['description'] = comment
if version != "1.0" and not family_obj and comment:
family_obj["description"] = comment
self.add_family(
path,
name,
@ -679,7 +698,11 @@ class ParserVariable:
# it's a dict, so a new variables!
continue
# 'variable' for compatibility to format 1.0
if key == "variable" and obj.get("type") != "dynamic" and obj.get("_type") != "dynamic":
if (
key == "variable"
and obj.get("type") != "dynamic"
and obj.get("_type") != "dynamic"
):
continue
if key in self.family_attrs:
yield key
@ -724,17 +747,20 @@ class ParserVariable:
del family["variable"]
# FIXME only for 1.0
if "variable" in family:
family['dynamic'] = {'type': 'variable',
'variable': family['variable'],
'propertyerror': False,
'allow_none': True,
family["dynamic"] = {
"type": "variable",
"variable": family["variable"],
"propertyerror": False,
"allow_none": True,
}
del family['variable']
del family["variable"]
if version != "1.0":
warning = f'"variable" attribute in dynamic family "{ path }" is depreciated in {filename}'
warn(warning)
if "variable" in family:
raise Exception(f'dynamic family must not have "variable" attribute for "{family["path"]}" in {family["xmlfiles"]}')
raise Exception(
f'dynamic family must not have "variable" attribute for "{family["path"]}" in {family["xmlfiles"]}'
)
else:
family_obj = self.family
# convert to Calculation objects
@ -969,7 +995,7 @@ class ParserVariable:
parent_dynamic,
)
self.variables.append(variable["path"])
if '.' in variable["path"]:
if "." in variable["path"]:
parent_path = variable["path"].rsplit(".", 1)[0]
else:
parent_path = "."
@ -987,10 +1013,10 @@ class ParserVariable:
del self.paths[path]
self.families.remove(path)
del self.parents[path]
if '.' in path:
if "." in path:
parent = path.rsplit(".", 1)[0]
else:
parent = '.'
parent = "."
self.parents[parent].remove(path)
###############################################################################################
@ -1003,9 +1029,7 @@ class ParserVariable:
):
"""Set Tiramisu object name"""
self.index += 1
self.reflector_names[
obj.path
] = f'{option_prefix}{self.index}{self.suffix}'
self.reflector_names[obj.path] = f"{option_prefix}{self.index}{self.suffix}"
###############################################################################################
# calculations
@ -1024,12 +1048,12 @@ class ParserVariable:
calculations = calculations[1]
if not isinstance(value, dict) or attribute not in calculations:
return False
if 'type' in value:
return value['type'] in CALCULATION_TYPES
if "type" in value:
return value["type"] in CALCULATION_TYPES
# auto set type
typ = set(CALCULATION_TYPES) & set(value)
if len(typ) == 1:
value['type'] = list(typ)[0]
value["type"] = list(typ)[0]
return True
return False
@ -1068,7 +1092,7 @@ class ParserVariable:
# auto set type
param_typ = set(CALCULATION_TYPES) & set(val)
if len(param_typ) == 1:
val['type'] = list(param_typ)[0]
val["type"] = list(param_typ)[0]
if not isinstance(val, dict) or "type" not in val:
param_typ = "any"
val = {
@ -1087,7 +1111,9 @@ class ParserVariable:
params.append(PARAM_TYPES[param_typ](**val))
except ValidationError as err:
raise DictConsistencyError(
f'"{attribute}" has an invalid "{key}" for {path}: {err}', 29, xmlfiles
f'"{attribute}" has an invalid "{key}" for {path}: {err}',
29,
xmlfiles,
) from err
calculation_object["params"] = params
#
@ -1164,11 +1190,11 @@ class RougailConvert(ParserVariable):
self.add_family(
n_path_prefix,
n_path_prefix,
{'description': path_prefix},
{"description": path_prefix},
"",
False,
None,
'',
"",
)
else:
root_parent = "."
@ -1196,12 +1222,13 @@ class RougailConvert(ParserVariable):
for idx, filename in enumerate(self.get_sorted_filename(extra_dirs)):
if not idx:
self.parse_family(
'',
"",
self.namespace,
namespace_path,
{'description': namespace,
{
"description": namespace,
},
'',
"",
)
self.parse_variable_file(
filename,
@ -1210,7 +1237,7 @@ class RougailConvert(ParserVariable):
else:
self.namespace = None
if root_parent == ".":
namespace_path = ''
namespace_path = ""
else:
namespace_path = f"{root_parent}"
if namespace_path in self.parents:
@ -1250,13 +1277,15 @@ class RougailConvert(ParserVariable):
)
if objects is None:
return
self.parse_root_file(filename,
self.parse_root_file(
filename,
path,
version,
objects,
)
def parse_root_file(self,
def parse_root_file(
self,
filename: str,
path: str,
version: str,
@ -1367,5 +1396,5 @@ class RougailConvert(ParserVariable):
if filename:
with open(filename, "w", encoding="utf-8") as tiramisu:
tiramisu.write(output)
#print(output)
# print(output)
return output

5
src/rougail/error.py
View file

@ -27,6 +27,7 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from .i18n import _
@ -74,13 +75,17 @@ class DictConsistencyError(Exception):
class UpgradeError(Exception):
"""Error during XML upgrade"""
## ---- generic exceptions ----
class NotFoundError(Exception):
"not found error"
pass
## ---- specific exceptions ----
class VariableCalculationDependencyError(Exception):
pass

1
src/rougail/i18n.py
View file

@ -26,6 +26,7 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
import gettext
import os
import sys

26
src/rougail/object_model.py
View file

@ -333,7 +333,7 @@ class JinjaCalculation(Calculation):
"type": "variable",
"variable": sub_variable,
}
if self.version != '1.0':
if self.version != "1.0":
default["params"][true_path]["propertyerror"] = False
if identifier:
default["params"][true_path]["identifier"] = identifier
@ -405,7 +405,8 @@ class _VariableCalculation(Calculation):
propertyerror: bool = True
allow_none: bool = False
def get_variable(self,
def get_variable(
self,
objectspace,
) -> "Variable":
if self.ori_path is None:
@ -467,7 +468,7 @@ class _VariableCalculation(Calculation):
calc_variable_is_multi = True
else:
calc_variable_is_multi = True
elif identifier and '{{ identifier }}' in identifier:
elif identifier and "{{ identifier }}" in identifier:
calc_variable_is_multi = True
if needs_multi:
if calc_variable_is_multi:
@ -481,12 +482,15 @@ class _VariableCalculation(Calculation):
msg = f'the variable "{self.path}" has an invalid attribute "{self.attribute_name}", it\'s a list'
raise DictConsistencyError(msg, 23, self.xmlfiles)
elif calc_variable_is_multi:
if variable.multi or variable.path.rsplit('.', 1)[0] != self.path.rsplit('.', 1)[0]:
if (
variable.multi
or variable.path.rsplit(".", 1)[0] != self.path.rsplit(".", 1)[0]
):
# it's not a follower or not in same leadership
msg = f'the variable "{self.path}" has an invalid attribute "{self.attribute_name}", the variable "{variable.path}" is a multi'
raise DictConsistencyError(msg, 21, self.xmlfiles)
else:
params[None][0]['index'] = {'index': {'type': 'index'}}
params[None][0]["index"] = {"index": {"type": "index"}}
return params
@ -504,7 +508,8 @@ class VariableCalculation(_VariableCalculation):
variable, identifier = self.get_variable(objectspace)
if not variable and self.optional:
raise VariableCalculationDependencyError()
params = self.get_params(objectspace,
params = self.get_params(
objectspace,
variable,
identifier,
)
@ -524,10 +529,12 @@ class VariablePropertyCalculation(_VariableCalculation):
objectspace,
) -> dict:
variable, identifier = self.get_variable(objectspace)
params = self.get_params(objectspace,
params = self.get_params(
objectspace,
variable,
identifier,
needs_multi=False,)
needs_multi=False,
)
variable = params[None][0]["variable"]
if self.when is not undefined:
if self.version == "1.0":
@ -650,7 +657,8 @@ class IdentifierPropertyCalculation(_IdentifierCalculation):
else:
msg = f'the identifier has an invalid attribute "{self.attribute_name}", when and when_not cannot set together'
raise DictConsistencyError
params = {None: [self.attribute_name, self.get_identifier()],
params = {
None: [self.attribute_name, self.get_identifier()],
"when": when,
"inverse": inverse,
}

1
src/rougail/providersupplier.py
View file

@ -16,6 +16,7 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
# from rougail.objspace import get_variables
# from rougail.utils import normalize_family
#

50
src/rougail/structural_commandline/annotator.py
View file

@ -19,12 +19,15 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from rougail.annotator.variable import Walk
from rougail.utils import _
from rougail.error import DictConsistencyError
class Annotator(Walk):
"""Annotate value"""
level = 80
def __init__(self, objectspace, *args) -> None:
@ -37,9 +40,9 @@ class Annotator(Walk):
if family.commandline:
continue
self.not_for_commandline(family)
not_for_commandlines.append(family.path + '.')
not_for_commandlines.append(family.path + ".")
for variable in self.get_variables():
if variable.type == 'symlink':
if variable.type == "symlink":
continue
variable_path = variable.path
for family_path in not_for_commandlines:
@ -53,37 +56,58 @@ class Annotator(Walk):
self.manage_negative_description(variable)
def not_for_commandline(self, variable) -> None:
self.objectspace.properties.add(variable.path, 'not_for_commandline', True)
self.objectspace.properties.add(variable.path, "not_for_commandline", True)
def manage_alternative_name(self, variable) -> None:
if not variable.alternative_name:
return
alternative_name = variable.alternative_name
variable_path = variable.path
all_letters = ''
all_letters = ""
for letter in alternative_name:
all_letters += letter
if all_letters == 'h':
if all_letters == "h":
msg = _(f'alternative_name "{alternative_name}" conflict with "--help"')
raise DictConsistencyError(msg, 202, variable.xmlfiles)
if all_letters in self.alternative_names:
msg = _(f'conflict alternative_name "{alternative_name}": "{variable_path}" and "{self.alternative_names[all_letters]}"')
msg = _(
f'conflict alternative_name "{alternative_name}": "{variable_path}" and "{self.alternative_names[all_letters]}"'
)
raise DictConsistencyError(msg, 202, variable.xmlfiles)
self.alternative_names[alternative_name] = variable_path
if '.' not in variable_path:
if "." not in variable_path:
path = alternative_name
else:
path = variable_path.rsplit('.', 1)[0] + '.' + alternative_name
self.objectspace.add_variable(alternative_name, {'type': 'symlink', 'path': path, 'opt': variable}, variable.xmlfiles, False, False, variable.version)
path = variable_path.rsplit(".", 1)[0] + "." + alternative_name
self.objectspace.add_variable(
alternative_name,
{"type": "symlink", "path": path, "opt": variable},
variable.xmlfiles,
False,
False,
variable.version,
)
def manage_negative_description(self, variable) -> None:
if not variable.negative_description:
if variable.type == 'boolean' and not self.objectspace.add_extra_options:
raise DictConsistencyError(_(f'negative_description is mandatory for boolean variable, but "{variable.path}" hasn\'t'), 200, variable.xmlfiles)
if variable.type == "boolean" and not self.objectspace.add_extra_options:
raise DictConsistencyError(
_(
f'negative_description is mandatory for boolean variable, but "{variable.path}" hasn\'t'
),
200,
variable.xmlfiles,
)
return
if variable.type != 'boolean':
raise DictConsistencyError(_(f'negative_description is only available for boolean variable, but "{variable.path}" is "{variable.type}"'), 201, variable.xmlfiles)
if variable.type != "boolean":
raise DictConsistencyError(
_(
f'negative_description is only available for boolean variable, but "{variable.path}" is "{variable.type}"'
),
201,
variable.xmlfiles,
)
self.objectspace.informations.add(
variable.path, "negative_description", variable.negative_description
)

19
src/rougail/structural_commandline/config.py
View file

@ -20,9 +20,12 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
def get_rougail_config(*,
def get_rougail_config(
*,
backward_compatibility=True,
) -> dict:
) -> dict:
options = """
structural_commandline:
description: Configuration rougail-structural_commandline
@ -31,12 +34,12 @@ structural_commandline:
description: Add extra options to tiramisu-cmdline-parser
default: true
"""
return {'name': 'exporter',
'process': 'structural',
'options': options,
'level': 20,
return {
"name": "exporter",
"process": "structural",
"options": options,
"level": 20,
}
__all__ = ('get_rougail_config')
__all__ = "get_rougail_config"

11
src/rougail/structural_commandline/object_model.py
View file

@ -19,18 +19,19 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from typing import Optional
from pydantic import BaseModel
class Variable(BaseModel):
alternative_name: Optional[str]=None
commandline: bool=True
negative_description: Optional[str]=None
alternative_name: Optional[str] = None
commandline: bool = True
negative_description: Optional[str] = None
class Family(BaseModel):
commandline: bool=True
commandline: bool = True
__all__ = ('Variable', 'Family')
__all__ = ("Variable", "Family")

83
src/rougail/tiramisu.py
View file

@ -27,13 +27,18 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from typing import Any
try:
from tiramisu5 import DynOptionDescription, calc_value
except ModuleNotFoundError:
from tiramisu import DynOptionDescription, calc_value
from importlib.machinery import SourceFileLoader as _SourceFileLoader
from importlib.util import spec_from_loader as _spec_from_loader, module_from_spec as _module_from_spec
from importlib.util import (
spec_from_loader as _spec_from_loader,
module_from_spec as _module_from_spec,
)
from jinja2 import StrictUndefined, DictLoader
from jinja2.sandbox import SandboxedEnvironment
from rougail.object_model import CONVERT_OPTION
@ -43,16 +48,16 @@ from tiramisu.error import ValueWarning, ConfigError, PropertiesOptionError
from .utils import normalize_family
global func
dict_env = {}
ENV = SandboxedEnvironment(loader=DictLoader(dict_env), undefined=StrictUndefined)
func = ENV.filters
ENV.compile_templates('jinja_caches', zip=None)
ENV.compile_templates("jinja_caches", zip=None)
class JinjaError:
__slot__ = ('_err',)
__slot__ = ("_err",)
def __init__(self, err):
self._err = err
@ -85,17 +90,17 @@ def test_propertyerror(value: Any) -> bool:
return isinstance(value, JinjaError)
ENV.tests['propertyerror'] = test_propertyerror
ENV.tests["propertyerror"] = test_propertyerror
def load_functions(path):
global _SourceFileLoader, _spec_from_loader, _module_from_spec, func
loader = _SourceFileLoader('func', path)
loader = _SourceFileLoader("func", path)
spec = _spec_from_loader(loader.name, loader)
func_ = _module_from_spec(spec)
loader.exec_module(func_)
for function in dir(func_):
if function.startswith('_'):
if function.startswith("_"):
continue
func[function] = getattr(func_, function)
@ -108,37 +113,52 @@ def rougail_calc_value(*args, __default_value=None, **kwargs):
@function_waiting_for_error
def jinja_to_function(__internal_variable, __internal_attribute, __internal_jinja, __internal_type, __internal_multi, __internal_files, __default_value=None, **kwargs):
def jinja_to_function(
__internal_variable,
__internal_attribute,
__internal_jinja,
__internal_type,
__internal_multi,
__internal_files,
__default_value=None,
**kwargs,
):
global ENV, CONVERT_OPTION
kw = {}
for key, value in kwargs.items():
if isinstance(value, PropertiesOptionError):
value = JinjaError(value)
if '.' in key:
if "." in key:
c_kw = kw
path, var = key.rsplit('.', 1)
for subkey in path.split('.'):
path, var = key.rsplit(".", 1)
for subkey in path.split("."):
c_kw = c_kw.setdefault(subkey, {})
c_kw[var] = value
else:
if key in kw:
raise ConfigError(f'internal error, multi key for "{key}" in jinja_to_function')
raise ConfigError(
f'internal error, multi key for "{key}" in jinja_to_function'
)
kw[key] = value
try:
values = ENV.get_template(__internal_jinja).render(kw, **func).strip()
except Exception as err:
raise ConfigError(f'cannot calculating "{__internal_attribute}" attribute for variable "{__internal_variable}" in {display_xmlfiles(__internal_files)}: {err}') from err
convert = CONVERT_OPTION[__internal_type].get('func', str)
raise ConfigError(
f'cannot calculating "{__internal_attribute}" attribute for variable "{__internal_variable}" in {display_xmlfiles(__internal_files)}: {err}'
) from err
convert = CONVERT_OPTION[__internal_type].get("func", str)
if __internal_multi:
values = [convert(val) for val in values.split('\n') if val != ""]
values = [convert(val) for val in values.split("\n") if val != ""]
if not values and __default_value is not None:
return __default_value
return values
try:
values = convert(values)
except Exception as err:
raise ConfigError(f'cannot converting "{__internal_attribute}" attribute for variable "{__internal_variable}" in {display_xmlfiles(__internal_files)}: {err}') from err
values = values if values != '' and values != 'None' else None
raise ConfigError(
f'cannot converting "{__internal_attribute}" attribute for variable "{__internal_variable}" in {display_xmlfiles(__internal_files)}: {err}'
) from err
values = values if values != "" and values != "None" else None
if values is None and __default_value is not None:
return __default_value
return values
@ -156,20 +176,20 @@ def variable_to_property(prop, value, when, inverse):
@function_waiting_for_error
def jinja_to_property(prop, when, inverse, **kwargs):
value = func['jinja_to_function'](**kwargs)
return func['variable_to_property'](prop, value is not None, when, inverse)
value = func["jinja_to_function"](**kwargs)
return func["variable_to_property"](prop, value is not None, when, inverse)
@function_waiting_for_error
def jinja_to_property_help(prop, **kwargs):
value = func['jinja_to_function'](**kwargs)
return (prop, f'\"{prop}\" ({value})')
value = func["jinja_to_function"](**kwargs)
return (prop, f'"{prop}" ({value})')
@function_waiting_for_error
def valid_with_jinja(warnings_only=False, **kwargs):
global ValueWarning
value = func['jinja_to_function'](**kwargs)
value = func["jinja_to_function"](**kwargs)
if value:
if warnings_only:
raise ValueWarning(value)
@ -177,12 +197,12 @@ def valid_with_jinja(warnings_only=False, **kwargs):
raise ValueError(value)
func['calc_value'] = rougail_calc_value
func['jinja_to_function'] = jinja_to_function
func['jinja_to_property'] = jinja_to_property
func['jinja_to_property_help'] = jinja_to_property_help
func['variable_to_property'] = variable_to_property
func['valid_with_jinja'] = valid_with_jinja
func["calc_value"] = rougail_calc_value
func["jinja_to_function"] = jinja_to_function
func["jinja_to_property"] = jinja_to_property
func["jinja_to_property_help"] = jinja_to_property_help
func["variable_to_property"] = variable_to_property
func["valid_with_jinja"] = valid_with_jinja
class ConvertDynOptionDescription(DynOptionDescription):
@ -213,9 +233,12 @@ class ConvertDynOptionDescription(DynOptionDescription):
def impl_get_display_name(
self,
subconfig,
with_quote: bool=False,
with_quote: bool = False,
) -> str:
display = super().impl_get_display_name(subconfig, with_quote=with_quote)
if "{{ identifier }}" in display:
return display.replace("{{ identifier }}", self.convert_identifier_to_path(self.get_identifiers(subconfig)[-1]))
return display.replace(
"{{ identifier }}",
self.convert_identifier_to_path(self.get_identifiers(subconfig)[-1]),
)
return display

32
src/rougail/tiramisureflector.py
View file

@ -28,6 +28,7 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from typing import Optional, Union
from json import dumps
from os.path import isfile, basename
@ -90,11 +91,14 @@ class TiramisuReflector:
self.text["header"].append(f"load_functions('{funcs_path}')")
if self.objectspace.export_with_import:
if objectspace.main_namespace:
self.text["header"].extend(["try:",
self.text["header"].extend(
[
"try:",
" groups.namespace",
"except:",
" groups.addgroup('namespace')",
])
]
)
for mode in self.objectspace.modes_level:
self.text["header"].append(f'ALLOWED_LEADER_PROPERTIES.add("{mode}")')
self.make_tiramisu_objects()
@ -112,9 +116,9 @@ class TiramisuReflector:
def make_tiramisu_objects(self) -> None:
"""make tiramisu objects"""
baseelt = BaseElt()
self.objectspace.reflector_names[
baseelt.path
] = f"option_0{self.objectspace.suffix}"
self.objectspace.reflector_names[baseelt.path] = (
f"option_0{self.objectspace.suffix}"
)
basefamily = Family(
baseelt,
self,
@ -340,7 +344,7 @@ class Common:
param.get("propertyerror", True),
param.get("identifier"),
param.get("dynamic"),
param.get('whole', False),
param.get("whole", False),
)
if param["type"] == "any":
if isinstance(param["value"], str):
@ -375,7 +379,7 @@ class Common:
if isinstance(ident, str):
ident = self.convert_str(ident)
identifiers.append(str(ident))
params.append('[' + ', '.join(identifiers) + ']')
params.append("[" + ", ".join(identifiers) + "]")
else:
param_type = "ParamOption"
if not propertyerror:
@ -475,13 +479,15 @@ class Variable(Common):
keys["values"] = self.populate_calculation(
self.elt.choices, return_a_tuple=True
)
if self.elt.type == 'regexp':
self.object_type = 'Regexp_' + self.option_name
self.tiramisu.text['header'].append(f'''class {self.object_type}(RegexpOption):
if self.elt.type == "regexp":
self.object_type = "Regexp_" + self.option_name
self.tiramisu.text["header"].append(
f"""class {self.object_type}(RegexpOption):
__slots__ = tuple()
_type = 'value'
{self.object_type}._regexp = re_compile(r"{self.elt.regexp}")
''')
"""
)
if self.elt.path in self.objectspace.multis:
keys["multi"] = self.objectspace.multis[self.elt.path]
if hasattr(self.elt, "default") and self.elt.default is not None:
@ -528,8 +534,8 @@ class Family(Common):
self.object_type = "Leadership"
else:
self.object_type = "OptionDescription"
if hasattr(self.elt, 'name') and self.elt.name == self.elt.namespace:
self.group_type = 'groups.namespace'
if hasattr(self.elt, "name") and self.elt.name == self.elt.namespace:
self.group_type = "groups.namespace"
else:
self.group_type = None
self.children = []

1
src/rougail/update/__init__.py
View file

@ -18,4 +18,5 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from .update import RougailUpgrade

136
src/rougail/update/update.py
View file

@ -63,7 +63,7 @@ class upgrade_010_to_10:
xmlsrc: str,
) -> None:
self.xmlsrc = xmlsrc
self.paths = {"family": {}, "variable": {}, 'dynamic': {}}
self.paths = {"family": {}, "variable": {}, "dynamic": {}}
self.lists = {
"service": {},
"ip": {},
@ -82,18 +82,26 @@ class upgrade_010_to_10:
sub_path: str,
true_sub_path: str,
*,
root: bool=False,
is_dynamic: bool=False,
root: bool = False,
is_dynamic: bool = False,
) -> dict:
new_families = {}
if root:
new_families['version'] = None
new_families["version"] = None
if "variables" in family:
for subelt in family["variables"]:
for typ, obj in subelt.items():
for subobj in obj:
local_is_dynamic = is_dynamic or subobj.get('dynamic') is not None
getattr(self, f"convert_{typ}")(subobj, new_families, sub_path, true_sub_path, local_is_dynamic)
local_is_dynamic = (
is_dynamic or subobj.get("dynamic") is not None
)
getattr(self, f"convert_{typ}")(
subobj,
new_families,
sub_path,
true_sub_path,
local_is_dynamic,
)
family.pop("variables")
return new_families
@ -112,7 +120,7 @@ class upgrade_010_to_10:
else:
true_sub_path = name
if is_dynamic:
name += '{{ suffix }}'
name += "{{ suffix }}"
if sub_path:
sub_path = sub_path + "." + name
else:
@ -126,7 +134,9 @@ class upgrade_010_to_10:
if typ == "dynamic":
family["variable"] = self.get_variable_path(value)
# add sub families and sub variables
sub_families = self.parse_variables(family, sub_path, true_sub_path, is_dynamic=is_dynamic)
sub_families = self.parse_variables(
family, sub_path, true_sub_path, is_dynamic=is_dynamic
)
for sub_name, sub_family in sub_families.copy().items():
if sub_name not in family:
continue
@ -152,13 +162,13 @@ class upgrade_010_to_10:
else:
true_sub_path = name
self.flatten_paths["variable"][name] = true_sub_path
name += '{{ suffix }}'
name += "{{ suffix }}"
if sub_path:
sub_path = sub_path + "." + name
else:
sub_path = name
if is_dynamic:
self.paths['dynamic'][true_sub_path] = sub_path
self.paths["dynamic"][true_sub_path] = sub_path
new_families[name] = variable
self.flatten_paths["variable"][name] = sub_path
self.paths["variable"][sub_path] = variable
@ -198,17 +208,18 @@ class upgrade_010_to_10:
)(test)
)
variable["test"] = tests
if variable.get('mandatory', False):
if variable.get("mandatory", False):
del variable["mandatory"]
CONVERT_TYPE = {'filename': 'unix_filename',
'password': 'secret',
CONVERT_TYPE = {
"filename": "unix_filename",
"password": "secret",
}
if variable.get('type') in CONVERT_TYPE:
variable['type'] = CONVERT_TYPE[variable['type']]
if variable.get("type") in CONVERT_TYPE:
variable["type"] = CONVERT_TYPE[variable["type"]]
def parse_variables_with_path(self):
for variable in self.paths["variable"].values():
multi = variable.get('multi', False)
multi = variable.get("multi", False)
if "value" in variable:
default = variable.pop("value")
if default is not None:
@ -223,7 +234,8 @@ class upgrade_010_to_10:
variable["choices"] = variable.pop("choice")
else:
variable["choices"] = [
self.get_value(choice, multi) for choice in variable.pop("choice")
self.get_value(choice, multi)
for choice in variable.pop("choice")
]
def parse_services(
@ -350,21 +362,21 @@ class upgrade_010_to_10:
if variable_path is None:
continue
variable = self.paths["variable"][variable_path]
if variable.get('multi', False):
if variable.get("multi", False):
multi = True
if apply_on_fallback:
condition_value = True
else:
if "{{ suffix }}" in source:
force_param = {'__var': source}
source = '__var'
force_param = {"__var": source}
source = "__var"
else:
force_param = None
condition_value = self.params_condition_to_jinja(
prop, source, condition["param"], name.endswith("if_in"), multi
)
if force_param:
condition_value.setdefault('params', {}).update(force_param)
condition_value.setdefault("params", {}).update(force_param)
for target in condition["target"]:
typ = target.get("type", "variable")
if typ == "variable":
@ -417,7 +429,9 @@ class upgrade_010_to_10:
check["param"] = [
{"text": variable_path, "type": "variable"}
] + check.get("param", [])
check_value = self.convert_param_function(check, variable.get('multi', False))
check_value = self.convert_param_function(
check, variable.get("multi", False)
)
variable.setdefault("validators", []).append(check_value)
def parse_fill(
@ -437,12 +451,16 @@ class upgrade_010_to_10:
"jinja": fill["name"],
}
else:
fill_value = self.convert_param_function(fill, variable.get('multi', False))
fill_value = self.convert_param_function(
fill, variable.get("multi", False)
)
variable["default"] = fill_value
if variable.get('mandatory') is False:
del variable['mandatory']
if variable.get("mandatory") is False:
del variable["mandatory"]
else:
raise Exception(f'cannot set fill to unknown variable "{variable_path}"')
raise Exception(
f'cannot set fill to unknown variable "{variable_path}"'
)
def params_condition_to_jinja(
self,
@ -538,8 +556,8 @@ class upgrade_010_to_10:
new_param = {attr_name: value}
value = attr_name
elif typ in ["index", "suffix"]:
if 'name' in value:
attr_name = value['name']
if "name" in value:
attr_name = value["name"]
else:
attr_name = f"__{typ}"
new_param = {attr_name: {"type": typ}}
@ -550,7 +568,7 @@ class upgrade_010_to_10:
value = value[typ]
elif "{{ suffix }}" in value[typ]:
path = value[typ]
attr_name = path.split('.')[-1][:-12] # remove {{ suffix }}
attr_name = path.split(".")[-1][:-12] # remove {{ suffix }}
new_param = {attr_name: value}
value = attr_name
else:
@ -568,16 +586,22 @@ class upgrade_010_to_10:
) -> str:
text = param["name"]
params = {}
# multi = False
# multi = False
if "param" in param and param["param"]:
if text == 'calc_value' and len(param["param"]) == 1 and isinstance(param["param"][0], dict) and param["param"][0].get('type') == 'variable' and param["param"][0].get("text"):
if (
text == "calc_value"
and len(param["param"]) == 1
and isinstance(param["param"][0], dict)
and param["param"][0].get("type") == "variable"
and param["param"][0].get("text")
):
value = param["param"][0]["text"]
path = self.get_variable_path(value)
if not path:
path = value
ret = {"type": "variable", "variable": path}
if 'optional' in param["param"][0]:
ret['optional'] = param["param"][0]["optional"]
if "optional" in param["param"][0]:
ret["optional"] = param["param"][0]["optional"]
return ret
first, *others = param["param"]
new_param, first = self.get_jinja_param_and_value(first, multi)
@ -604,9 +628,13 @@ class upgrade_010_to_10:
if not multi:
text = "{{ " + text + " }}"
else:
text = """{% for __variable in """ + text + """ %}
text = (
"""{% for __variable in """
+ text
+ """ %}
{{ __variable }}
{% endfor %}"""
)
ret = {"type": "jinja", "jinja": text}
if params:
ret["params"] = params
@ -659,16 +687,21 @@ class RougailUpgrade:
def run(
self,
):
for dict_dir, dest_dir in zip(self.rougailconfig["main_dictionaries"], self.rougailconfig["upgrade_options.main_dictionaries"]):
for dict_dir, dest_dir in zip(
self.rougailconfig["main_dictionaries"],
self.rougailconfig["upgrade_options.main_dictionaries"],
):
self._load_dictionaries(
dict_dir,
dest_dir,
normalize_family(self.rougailconfig["main_namespace"]),
)
if self.rougailconfig['main_namespace']:
if self.rougailconfig["main_namespace"]:
if self.rougailconfig["extra_dictionaries"]:
dst_extra_dir = self.rougailconfig["upgrade_options.extra_dictionary"]
for namespace, extra_dirs in self.rougailconfig["extra_dictionaries"].items():
for namespace, extra_dirs in self.rougailconfig[
"extra_dictionaries"
].items():
extra_dstsubfolder = Path(dst_extra_dir) / namespace
if not extra_dstsubfolder.is_dir():
extra_dstsubfolder.mkdir()
@ -697,7 +730,7 @@ class RougailUpgrade:
for filename in filenames:
xmlsrc = Path(srcfolder) / Path(filename)
ymldst = Path(dstfolder) / (Path(filename).stem + '.yml')
ymldst = Path(dstfolder) / (Path(filename).stem + ".yml")
if filename.endswith(".xml"):
if parse is None:
raise Exception("XML module is not installed")
@ -732,7 +765,7 @@ class RougailUpgrade:
root_services = root_services_
if function_version == search_function_name:
function_found = True
if root != {'version': None}:
if root != {"version": None}:
root["version"] = float(version)
with ymldst.open("w") as ymlfh:
yaml = YAML()
@ -923,25 +956,26 @@ class RougailUpgrade:
"variable": value.pop("variable"),
"propertyerror": False,
}
if '{{ suffix }}' not in key:
new_root[key + '{{ suffix }}'] = new_root.pop(key)
if "{{ suffix }}" not in key:
new_root[key + "{{ suffix }}"] = new_root.pop(key)
update_root = True
self._update_1_1(value)
for typ, obj in {'boolean': bool,
'number': int,
'string': str,
'float': float,
for typ, obj in {
"boolean": bool,
"number": int,
"string": str,
"float": float,
}.items():
if value.get('type') == typ:
default = value.get('default')
if value.get("type") == typ:
default = value.get("default")
if default is None or default == []:
continue
if isinstance(default, obj):
del value['type']
del value["type"]
elif isinstance(default, list) and isinstance(default[0], obj):
del value['type']
if value.get('multi') and isinstance(value.get('default'), list):
del value['multi']
del value["type"]
if value.get("multi") and isinstance(value.get("default"), list):
del value["multi"]
if update_root:
root.clear()
root.update(new_root)

11
src/rougail/utils.py
View file

@ -27,6 +27,7 @@ You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from typing import List, Union
from unicodedata import normalize, combining
import re
@ -117,7 +118,9 @@ def get_jinja_variable_to_param(
for g in parsed_content.find_all(Getattr):
variables.add(recurse_getattr(g))
except TemplateSyntaxError as err:
msg = _(f'error in jinja "{jinja_text}" for the variable "{ current_path }": {err}')
msg = _(
f'error in jinja "{jinja_text}" for the variable "{ current_path }": {err}'
)
raise DictConsistencyError(msg, 39, xmlfiles) from err
variables = list(variables)
variables.sort(reverse=True)
@ -135,7 +138,7 @@ def get_jinja_variable_to_param(
if variable and variable.path in objectspace.variables:
founded_variables[variable_path] = (identifier, variable)
else:
sub_family = variable_path + '.'
sub_family = variable_path + "."
for founded_variable in chain(founded_variables, unknown_variables):
if founded_variable.startswith(sub_family):
break
@ -149,8 +152,8 @@ def get_jinja_variable_to_param(
else:
root_path = None
vpath = variable_path
while '.' in vpath:
vpath = vpath.rsplit('.', 1)[0]
while "." in vpath:
vpath = vpath.rsplit(".", 1)[0]
variable, identifier = objectspace.paths.get_with_dynamic(
vpath,
path_prefix,