diff --git a/tests/test_dyn_optiondescription.py b/tests/test_dyn_optiondescription.py index d287e3f..d14bc40 100644 --- a/tests/test_dyn_optiondescription.py +++ b/tests/test_dyn_optiondescription.py @@ -75,6 +75,26 @@ async def test_build_dyndescription(): assert not await list_sessions() +@pytest.mark.asyncio +async def test_build_dyndescription_with_int(): + int1 = IntOption('int', '', default=Calculation(calc_value, Params(ParamSuffix()))) + dod = DynOptionDescription('dod', '', [int1], suffixes=Calculation(return_list, Params(ParamValue([1, 2])))) + od1 = OptionDescription('od', '', [dod]) + async with await Config(od1) as cfg: + assert await cfg.value.dict() == {'dod1.int1': 1, 'dod2.int2': 2} + assert not await list_sessions() + + +@pytest.mark.asyncio +async def test_build_dyndescription_with_dot(): + st1 = StrOption('st', '', default=Calculation(calc_value, Params(ParamSuffix()))) + dod = DynOptionDescription('dod', '', [st1], suffixes=Calculation(return_list_dot)) + od1 = OptionDescription('od', '', [dod]) + async with await Config(od1) as cfg: + assert await cfg.value.dict() == {'dodval_1.stval_1': 'val.1', 'dodval_2.stval_2': 'val.2'} + assert not await list_sessions() + + @pytest.mark.asyncio async def test_build_dyndescription_raise(): st1 = StrOption('st', '') @@ -1138,6 +1158,72 @@ async def test_leadership_dyndescription(): assert not await list_sessions() +@pytest.mark.asyncio +async def test_leadership_dyndescription_force_store_value_leader(): + st1 = StrOption('st1', "", multi=True, default=Calculation(return_list), properties=('force_store_value',)) + st2 = StrOption('st2', "", multi=True, default=Calculation(return_list, Params(ParamOption(st1)))) + stm = Leadership('st1', '', [st1, st2]) + val1 = StrOption('val1', '', multi=True, default=['val1', 'val2']) + st = DynOptionDescription('st', '', [stm], suffixes=Calculation(return_list, Params(ParamOption(val1)))) + od = OptionDescription('od', '', [val1, st]) + od2 = OptionDescription('od', '', [od]) + async with await Config(od2) as cfg: + await cfg.property.read_write() + assert await cfg.option('od.stval1.st1val1.st1val1').owner.isdefault() == False + assert await cfg.option('od.stval2.st1val2.st1val2').owner.isdefault() == False + assert await cfg.option('od.stval1.st1val1.st2val1', 0).owner.isdefault() == True + assert await cfg.option('od.stval1.st1val1.st2val1', 1).owner.isdefault() == True + assert await cfg.option('od.stval2.st1val2.st2val2', 0).owner.isdefault() == True + assert await cfg.option('od.stval2.st1val2.st2val2', 1).owner.isdefault() == True + assert await cfg.value.dict() == {'od.stval1.st1val1.st1val1': ['val1', 'val2'], 'od.stval1.st1val1.st2val1': ['val1', 'val2'], 'od.stval2.st1val2.st1val2': ['val1', 'val2'], 'od.stval2.st1val2.st2val2': ['val1', 'val2'], 'od.val1': ['val1', 'val2']} + # + await cfg.option('od.val1').value.set(['val1', 'val2', 'val3']) + assert await cfg.option('od.stval3.st1val3.st1val3').owner.isdefault() == False + assert await cfg.option('od.stval3.st1val3.st2val3', 0).owner.isdefault() == True + assert await cfg.option('od.stval3.st1val3.st2val3', 1).owner.isdefault() == True + assert await cfg.value.dict() == {'od.stval1.st1val1.st1val1': ['val1', 'val2'], 'od.stval1.st1val1.st2val1': ['val1', 'val2'], 'od.stval2.st1val2.st1val2': ['val1', 'val2'], 'od.stval2.st1val2.st2val2': ['val1', 'val2'], 'od.stval3.st1val3.st1val3': ['val1', 'val2'], 'od.stval3.st1val3.st2val3': ['val1', 'val2'], 'od.val1': ['val1', 'val2', 'val3']} + # + await cfg.option('od.stval3.st1val3.st1val3').value.set(['val1', 'val2', 'val3']) + assert await cfg.option('od.stval3.st1val3.st1val3').owner.isdefault() == False + assert await cfg.option('od.stval3.st1val3.st2val3', 0).owner.isdefault() == True + assert await cfg.option('od.stval3.st1val3.st2val3', 1).owner.isdefault() == True + assert await cfg.option('od.stval3.st1val3.st2val3', 2).owner.isdefault() == True + assert await cfg.value.dict() == {'od.stval1.st1val1.st1val1': ['val1', 'val2'], 'od.stval1.st1val1.st2val1': ['val1', 'val2'], 'od.stval2.st1val2.st1val2': ['val1', 'val2'], 'od.stval2.st1val2.st2val2': ['val1', 'val2'], 'od.stval3.st1val3.st1val3': ['val1', 'val2', 'val3'], 'od.stval3.st1val3.st2val3': ['val1', 'val2', 'val3'], 'od.val1': ['val1', 'val2', 'val3']} + + +@pytest.mark.asyncio +async def test_leadership_dyndescription_force_store_value(): + st1 = StrOption('st1', "", multi=True, default=Calculation(return_list)) + st2 = StrOption('st2', "", multi=True, properties=('force_store_value',), default=Calculation(return_list, Params(ParamOption(st1)))) + stm = Leadership('st1', '', [st1, st2]) + val1 = StrOption('val1', '', multi=True, default=['val1', 'val2']) + st = DynOptionDescription('st', '', [stm], suffixes=Calculation(return_list, Params(ParamOption(val1)))) + od = OptionDescription('od', '', [val1, st]) + od2 = OptionDescription('od', '', [od]) + async with await Config(od2) as cfg: + await cfg.property.read_write() + assert await cfg.option('od.stval1.st1val1.st1val1').owner.isdefault() == True + assert await cfg.option('od.stval2.st1val2.st1val2').owner.isdefault() == True + assert await cfg.option('od.stval1.st1val1.st2val1', 0).owner.isdefault() == False + assert await cfg.option('od.stval1.st1val1.st2val1', 1).owner.isdefault() == False + assert await cfg.option('od.stval2.st1val2.st2val2', 0).owner.isdefault() == False + assert await cfg.option('od.stval2.st1val2.st2val2', 1).owner.isdefault() == False + assert await cfg.value.dict() == {'od.stval1.st1val1.st1val1': ['val1', 'val2'], 'od.stval1.st1val1.st2val1': ['val1', 'val2'], 'od.stval2.st1val2.st1val2': ['val1', 'val2'], 'od.stval2.st1val2.st2val2': ['val1', 'val2'], 'od.val1': ['val1', 'val2']} + # + await cfg.option('od.val1').value.set(['val1', 'val2', 'val3']) + assert await cfg.option('od.stval3.st1val3.st1val3').owner.isdefault() == True + assert await cfg.option('od.stval3.st1val3.st2val3', 0).owner.isdefault() == False + assert await cfg.option('od.stval3.st1val3.st2val3', 1).owner.isdefault() == False + assert await cfg.value.dict() == {'od.stval1.st1val1.st1val1': ['val1', 'val2'], 'od.stval1.st1val1.st2val1': ['val1', 'val2'], 'od.stval2.st1val2.st1val2': ['val1', 'val2'], 'od.stval2.st1val2.st2val2': ['val1', 'val2'], 'od.stval3.st1val3.st1val3': ['val1', 'val2'], 'od.stval3.st1val3.st2val3': ['val1', 'val2'], 'od.val1': ['val1', 'val2', 'val3']} + # + await cfg.option('od.stval3.st1val3.st1val3').value.set(['val1', 'val2', 'val3']) + assert await cfg.option('od.stval3.st1val3.st1val3').owner.isdefault() == False + assert await cfg.option('od.stval3.st1val3.st2val3', 0).owner.isdefault() == False + assert await cfg.option('od.stval3.st1val3.st2val3', 1).owner.isdefault() == False + assert await cfg.option('od.stval3.st1val3.st2val3', 2).owner.isdefault() == False + assert await cfg.value.dict() == {'od.stval1.st1val1.st1val1': ['val1', 'val2'], 'od.stval1.st1val1.st2val1': ['val1', 'val2'], 'od.stval2.st1val2.st1val2': ['val1', 'val2'], 'od.stval2.st1val2.st2val2': ['val1', 'val2'], 'od.stval3.st1val3.st1val3': ['val1', 'val2', 'val3'], 'od.stval3.st1val3.st2val3': ['val1', 'val2', 'val3'], 'od.val1': ['val1', 'val2', 'val3']} + + @pytest.mark.asyncio async def test_leadership_default_multi_dyndescription(): st1 = StrOption('st1', "", multi=True) diff --git a/tiramisu/autolib.py b/tiramisu/autolib.py index 9f49bc3..ceb613a 100644 --- a/tiramisu/autolib.py +++ b/tiramisu/autolib.py @@ -358,7 +358,10 @@ async def manager_callback(callbk: Param, suffix = callbk.suffix else: if not option.impl_is_dynsymlinkoption(): - msg = 'option "{}" is not dynamic but is an argument of the dynamic option "{}" in a callback' + if callbk_option.issubdyn(): + msg = 'internal error: option "{}" is dynamic but is not a DynSymlinkOption' + else: + msg = 'option "{}" is not dynamic but has an argument with the dynamic option "{}" in a callback' raise ConfigError(_(msg).format(option.impl_get_display_name(), callbk_option.impl_get_display_name(), )) diff --git a/tiramisu/config.py b/tiramisu/config.py index b7796e0..09476e7 100644 --- a/tiramisu/config.py +++ b/tiramisu/config.py @@ -83,7 +83,8 @@ class SubConfig: cconfig_bag) moption_bag.properties = await self.cfgimpl_get_settings().getproperties(moption_bag) value = await self.getattr(leaderpath, - moption_bag) + moption_bag, + ) self._impl_length = len(value) def cfgimpl_get_length(self): @@ -283,7 +284,8 @@ class SubConfig: option_bag, from_follower=False, needs_re_verify_follower_properties=False, - need_help=True): + need_help=True, + ): """ :return: option's value if name is an option name, OptionDescription otherwise @@ -350,7 +352,8 @@ class SubConfig: 'length ({})').format(option.impl_get_display_name(), follower_len, length, - option_bag.index)) + option_bag.index, + )) if option.impl_is_follower() and option_bag.index is None: value = [] diff --git a/tiramisu/option/dynoptiondescription.py b/tiramisu/option/dynoptiondescription.py index b79a98f..c38bc51 100644 --- a/tiramisu/option/dynoptiondescription.py +++ b/tiramisu/option/dynoptiondescription.py @@ -19,7 +19,7 @@ # the whole pypy projet is under MIT licence # ____________________________________________________________ import re -from typing import List, Callable +from typing import List, Callable, Any from itertools import chain from ..autolib import ParamOption @@ -72,7 +72,14 @@ class DynOptionDescription(OptionDescription): self._suffixes = suffixes def convert_suffix_to_path(self, - suffix): + suffix: Any, + ) -> str: + if suffix is None: + return None + if not isinstance(suffix, str): + suffix = str(suffix) + if '.' in suffix: + suffix = suffix.replace('.', '_') return suffix async def get_suffixes(self, @@ -84,28 +91,28 @@ class DynOptionDescription(OptionDescription): values = await self._suffixes.execute(option_bag) if values is None: values = [] + values_ = [] if __debug__: if not isinstance(values, list): raise ValueError(_('DynOptionDescription suffixes for option "{}", is not a list ({})' '').format(self.impl_get_display_name(), values)) - values_ = [] - for val in values: - val = self.convert_suffix_to_path(val) - if not isinstance(val, str) or re.match(NAME_REGEXP, val) is None: - if val is not None: - raise ValueError(_('invalid suffix "{}" for option "{}"' - '').format(val, - self.impl_get_display_name())) - else: - values_.append(val) + for val in values: + cval = self.convert_suffix_to_path(val) + if not isinstance(cval, str) or re.match(NAME_REGEXP, cval) is None: + if __debug__ and cval is not None: + raise ValueError(_('invalid suffix "{}" for option "{}"' + '').format(cval, + self.impl_get_display_name())) + else: + values_.append(val) + if __debug__: if len(values_) > len(set(values_)): extra_values = values_.copy() for val in set(values_): extra_values.remove(val) raise ValueError(_('DynOptionDescription suffixes return a list with multiple value ' '"{}"''').format(extra_values)) - values = values_ - return values + return values_ def impl_is_dynoptiondescription(self) -> bool: return True diff --git a/tiramisu/option/leadership.py b/tiramisu/option/leadership.py index c36824f..a03926d 100644 --- a/tiramisu/option/leadership.py +++ b/tiramisu/option/leadership.py @@ -139,27 +139,34 @@ class Leadership(OptionDescription): value, option_bag, owner, - dyn=None) -> None: + dyn=None, + ) -> None: settings = option_bag.config_bag.context.cfgimpl_get_settings() if value: - rgevalue = range(len(value)) if dyn is None: dyn = self - for follower in await dyn.get_children(option_bag.config_bag): + for idx, follower in enumerate(await dyn.get_children(option_bag.config_bag)): foption_bag = OptionBag() foption_bag.set_option(follower, None, option_bag.config_bag) if 'force_store_value' in await settings.getproperties(foption_bag): - for index in rgevalue: + if idx == 0: + indexes = [None] + else: + indexes = range(len(value)) + for index in indexes: foption_bag = OptionBag() foption_bag.set_option(follower, index, option_bag.config_bag) foption_bag.properties = await settings.getproperties(foption_bag) - await values._setvalue(foption_bag, - await values.getvalue(foption_bag), - owner) + await values._p_.setvalue(foption_bag.config_bag.connection, + foption_bag.path, + await values.getvalue(foption_bag), + owner, + index, + ) async def pop(self, values: Values, @@ -232,8 +239,10 @@ class Leadership(OptionDescription): def to_dynoption(self, rootpath: str, suffix: str, - ori_dyn) -> SynDynLeadership: + ori_dyn, + ) -> SynDynLeadership: return SynDynLeadership(self, rootpath, suffix, - ori_dyn) + ori_dyn, + ) diff --git a/tiramisu/option/option.py b/tiramisu/option/option.py index b9ea370..6117e2d 100644 --- a/tiramisu/option/option.py +++ b/tiramisu/option/option.py @@ -520,8 +520,10 @@ class Option(BaseOption): def to_dynoption(self, rootpath: str, suffix: str, - ori_dyn) -> SynDynOption: + ori_dyn, + ) -> SynDynOption: return SynDynOption(self, rootpath, suffix, - ori_dyn) + ori_dyn, + ) diff --git a/tiramisu/option/optiondescription.py b/tiramisu/option/optiondescription.py index 3fd0806..858f1ea 100644 --- a/tiramisu/option/optiondescription.py +++ b/tiramisu/option/optiondescription.py @@ -87,7 +87,7 @@ class CacheOptionDescription(BaseOption): if not option.impl_is_symlinkoption(): properties = option.impl_getproperties() if 'force_store_value' in properties: - force_store_values.append((subpath, option)) + force_store_values.append(option) if __debug__ and ('force_default_on_freeze' in properties or \ 'force_metaconfig_on_freeze' in properties) and \ 'frozen' not in properties and \ @@ -112,25 +112,59 @@ class CacheOptionDescription(BaseOption): async def impl_build_force_store_values(self, config_bag: ConfigBag, ) -> None: + async def do_option_bags(option): + if option.issubdyn(): + dynopt = option.getsubdyn() + rootpath = dynopt.impl_getpath() + subpaths = [rootpath] + option.impl_getpath()[len(rootpath) + 1:].split('.')[1:] + for suffix in await dynopt.get_suffixes(config_bag): + path_suffix = dynopt.convert_suffix_to_path(suffix) + subpath = '.'.join([subp + path_suffix for subp in subpaths]) + doption = option.to_dynoption(subpath, + suffix, + dynopt, + ) + doption_bag = OptionBag() + doption_bag.set_option(doption, + None, + config_bag, + ) + yield doption_bag + else: + option_bag = OptionBag() + option_bag.set_option(option, + None, + config_bag) + yield option_bag if 'force_store_value' not in config_bag.properties: return values = config_bag.context.cfgimpl_get_values() - for subpath, option in self._cache_force_store_values: - if not await values._p_.hasvalue(config_bag.connection, - subpath): - if option.impl_is_follower(): - option_bag = OptionBag() - leader = option.impl_get_leadership().get_leader() - option_bag.set_option(leader, - None, - config_bag) - option_bag.properties = frozenset() - follower_len = len(await values.getvalue(option_bag)) + for option in self._cache_force_store_values: + if option.impl_is_follower(): + leader = option.impl_get_leadership().get_leader() + async for leader_option_bag in do_option_bags(leader): + leader_option_bag.properties = frozenset() + follower_len = len(await values.getvalue(leader_option_bag)) + if option.issubdyn(): + subpath = leader_option_bag.option.rootpath + doption = option.to_dynoption(subpath, + leader_option_bag.option.impl_getsuffix(), + leader_option_bag.option.ori_dyn, + ) + else: + doption = option + subpath = doption.impl_getpath() for index in range(follower_len): + if await values._p_.hasvalue(config_bag.connection, + subpath, + index, + ): + continue option_bag = OptionBag() - option_bag.set_option(option, + option_bag.set_option(doption, index, - config_bag) + config_bag, + ) option_bag.properties = frozenset() value = await values.getvalue(option_bag) if value is None: @@ -141,41 +175,23 @@ class CacheOptionDescription(BaseOption): owners.forced, index, False) - else: - option_bags = [] - if option.issubdyn(): - dynopt = option.getsubdyn() - rootpath = dynopt.impl_getpath() - subpaths = [rootpath] + option.impl_getpath()[len(rootpath) + 1:].split('.')[1:] - for suffix in await dynopt.get_suffixes(config_bag): - path_suffix = dynopt.convert_suffix_to_path(suffix) - subpath = '.'.join([subp + path_suffix for subp in subpaths]) - doption = option.to_dynoption(subpath, - suffix, - option) - doption_bag = OptionBag() - doption_bag.set_option(doption, - None, - config_bag) - option_bags.append(doption_bag) - else: - option_bag = OptionBag() - option_bag.set_option(option, + else: + async for option_bag in do_option_bags(option): + option_bag.properties = frozenset() + value = await values.getvalue(option_bag) + if value is None: + continue + if await values._p_.hasvalue(config_bag.connection, + option_bag.option.impl_getpath(), + ): + continue + await values._p_.setvalue(config_bag.connection, + option_bag.path, + value, + owners.forced, None, - config_bag) - option_bags.append(option_bag) - for option_bag in option_bags: - option_bag.properties = frozenset() - value = await values.getvalue(option_bag) - if value is None: - continue - await values._p_.setvalue(config_bag.connection, - option_bag.path, - value, - owners.forced, - None, - False, - ) + False, + ) class OptionDescriptionWalk(CacheOptionDescription): diff --git a/tiramisu/option/syndynoption.py b/tiramisu/option/syndynoption.py index f893672..3adefe1 100644 --- a/tiramisu/option/syndynoption.py +++ b/tiramisu/option/syndynoption.py @@ -56,11 +56,11 @@ class SynDynOption: self.suffix == left.suffix def impl_getname(self) -> str: - return self.opt.impl_getname() + self.suffix + return self.opt.impl_getname() + self.ori_dyn.convert_suffix_to_path(self.suffix) def impl_get_display_name(self) -> str: return self.opt._get_display_name(dyn_name=self.impl_getname(), - suffix=self.suffix, + suffix=self.ori_dyn.convert_suffix_to_path(self.suffix), ) def impl_getsuffix(self) -> str: @@ -75,6 +75,8 @@ class SynDynOption: def impl_get_leadership(self): leadership = self.opt.impl_get_leadership() if leadership: - return leadership.to_dynoption(self.rootpath, + rootpath = self.rootpath.rsplit('.', 1)[0] + return leadership.to_dynoption(rootpath, self.suffix, - self.ori_dyn) + self.ori_dyn, + ) diff --git a/tiramisu/option/syndynoptiondescription.py b/tiramisu/option/syndynoptiondescription.py index 2e3539e..408bcec 100644 --- a/tiramisu/option/syndynoptiondescription.py +++ b/tiramisu/option/syndynoptiondescription.py @@ -49,10 +49,12 @@ class SynDynOptionDescription: self.ori_dyn = ori_dyn def __getattr__(self, - name: str) -> Any: + name: str, + ) -> Any: # if not in SynDynOptionDescription, get value in self.opt return getattr(self.opt, - name) + name, + ) def impl_getopt(self) -> BaseOption: return self.opt @@ -61,8 +63,9 @@ class SynDynOptionDescription: name: str, config_bag: ConfigBag, subpath: str) -> BaseOption: - if name.endswith(self._suffix): - oname = name[:-len(self._suffix)] + suffix = self.ori_dyn.convert_suffix_to_path(self._suffix) + if name.endswith(suffix): + oname = name[:-len(suffix)] try: child = self._children[1][self._children[0].index(oname)] except ValueError: @@ -71,13 +74,13 @@ class SynDynOptionDescription: else: return child.to_dynoption(subpath, self._suffix, - self.opt) + self.ori_dyn) raise AttributeError(_('unknown option "{0}" ' 'in dynamic optiondescription "{1}"' '').format(name, self.impl_get_display_name())) def impl_getname(self) -> str: - return self.opt.impl_getname() + self._suffix + return self.opt.impl_getname() + self.ori_dyn.convert_suffix_to_path(self._suffix) def impl_is_dynoptiondescription(self) -> bool: return True @@ -91,7 +94,8 @@ class SynDynOptionDescription: for child in await self.opt.get_children(config_bag): children.append(child.to_dynoption(subpath, self._suffix, - self.opt)) + self.ori_dyn, + )) return children def impl_is_dynsymlinkoption(self) -> bool: diff --git a/tiramisu/value.py b/tiramisu/value.py index 641e46b..0b577e3 100644 --- a/tiramisu/value.py +++ b/tiramisu/value.py @@ -138,7 +138,8 @@ class Values: return ret async def getvalue(self, - option_bag): + option_bag, + ): """actually retrieves the value :param path: the path of the `Option` @@ -159,7 +160,8 @@ class Values: option_bag.path, owners.default, index=_index, - with_value=True) + with_value=True, + ) if owner == owners.default or \ ('frozen' in option_bag.properties and \ ('force_default_on_freeze' in option_bag.properties or await self.force_to_metaconfig(option_bag))): @@ -181,7 +183,8 @@ class Values: return value async def getdefaultvalue(self, - option_bag): + option_bag, + ): """get default value: - get parents config value or - get calculated value or @@ -200,7 +203,8 @@ class Values: # now try to get default value: value = await self.calc_value(option_bag, - option_bag.option.impl_getdefault()) + option_bag.option.impl_getdefault(), + ) if option_bag.index is not None and isinstance(value, (list, tuple)): if value and option_bag.option.impl_is_submulti(): # first index is a list, assume other data are list too @@ -340,7 +344,8 @@ class Values: await option_bag.option.impl_get_leadership().follower_force_store_value(self, value, option_bag, - owners.forced) + owners.forced, + ) async def setvalue_validation(self, value, @@ -400,21 +405,33 @@ class Values: subpath = '.'.join([subp + path_suffix for subp in subpaths]) doption = coption.to_dynoption(subpath, suffix, - coption, + option, ) - coption_bag = OptionBag() - coption_bag.set_option(doption, - None, - option_bag.config_bag, - ) - coption_bag.properties = await settings.getproperties(coption_bag) - await self._p_.setvalue(coption_bag.config_bag.connection, - coption_bag.path, - await self.getvalue(coption_bag), - owners.forced, - None, - False, - ) + if coption.impl_is_follower(): + leader = coption.impl_get_leadership().get_leader() + loption_bag = OptionBag() + loption_bag.set_option(leader, + None, + option_bag.config_bag, + ) + loption_bag.properties = frozenset() + indexes = range(len(await self.getvalue(loption_bag))) + else: + indexes = [None] + for index in indexes: + coption_bag = OptionBag() + coption_bag.set_option(doption, + index, + option_bag.config_bag, + ) + coption_bag.properties = await settings.getproperties(coption_bag) + await self._p_.setvalue(coption_bag.config_bag.connection, + coption_bag.path, + await self.getvalue(coption_bag), + owners.forced, + index, + False, + ) async def _get_modified_parent(self, option_bag: OptionBag) -> Optional[OptionBag]: