less commit + better meta support

test/test_metaconfig.py

@@ -11,6 +11,10 @@ from tiramisu.error import ConfigError, ConflictError
 owners.addowner('meta')
 
 
+def return_value(value=None):
+    return value
+
+
 def raise_exception():
     raise Exception('test')
 
@@ -528,3 +532,70 @@ def test_meta_exception_meta():
     meta = MetaConfig([conf1, conf2])
     meta.read_write()
     raises(Exception, "conf1.make_dict()")
+
+
+def test_meta_callback():
+    val1 = StrOption('val1', "", 'val')
+    val2 = StrOption('val2', "", callback=return_value, callback_params={'': ((val1, False),)})
+    val3 = StrOption('val3', "", callback=return_value, callback_params={'': ('yes',)})
+    val4 = StrOption('val4', "", callback=return_value, callback_params={'value': ((val1, False),)})
+    val5 = StrOption('val5', "", callback=return_value, callback_params={'value': ('yes',)})
+    maconfig = OptionDescription('rootconfig', '', [val1, val2, val3, val4, val5])
+    cfg = Config(maconfig, name='cfg')
+    meta = MetaConfig([cfg])
+    meta.read_write()
+    assert meta.cfg.make_dict() == {'val3': 'yes', 'val2': 'val', 'val1': 'val', 'val5': 'yes', 'val4': 'val'}
+    meta.cfg.val1 = 'new'
+    assert meta.cfg.make_dict() == {'val3': 'yes', 'val2': 'new', 'val1': 'new', 'val5': 'yes', 'val4': 'new'}
+    del(meta.cfg.val1)
+    meta.val1 = 'new'
+    assert meta.cfg.make_dict() == {'val3': 'yes', 'val2': 'new', 'val1': 'new', 'val5': 'yes', 'val4': 'new'}
+    #del(meta.val1)
+    meta.cfg.val4 = 'new1'
+    assert meta.cfg.make_dict() == {'val3': 'yes', 'val2': 'new', 'val1': 'new', 'val5': 'yes', 'val4': 'new1'}
+    del(meta.cfg.val4)
+    meta.val4 = 'new1'
+    assert meta.cfg.make_dict() == {'val3': 'yes', 'val2': 'new', 'val1': 'new', 'val5': 'yes', 'val4': 'new1'}
+    del(meta.val4)
+
+
+def test_meta_callback_slave():
+    val = StrOption('val', "", default='val')
+    val1 = StrOption('val1', "", multi=True, callback=return_value, callback_params={'': ((val, False),)})
+    val3 = StrOption('val2', "", multi=True, callback=return_value, callback_params={'': ((val1, False),)})
+    val4 = StrOption('val3', "", multi=True, callback=return_value, callback_params={'': ((val1, False),)})
+    interface1 = OptionDescription('val1', '', [val1, val3, val4])
+    interface1.impl_set_group_type(groups.master)
+    maconfig = OptionDescription('rootconfig', '', [val, interface1])
+    cfg = Config(maconfig, name='cfg')
+    meta = MetaConfig([cfg])
+    meta.read_write()
+    assert meta.cfg.make_dict() == {'val1.val2': ['val'], 'val1.val1': ['val'], 'val1.val3': ['val'], 'val': 'val'}
+    meta.cfg.val = 'val1'
+    assert meta.cfg.make_dict() == {'val1.val2': ['val1'], 'val1.val1': ['val1'], 'val1.val3': ['val1'], 'val': 'val1'}
+    del(meta.cfg.val)
+    meta.val = 'val1'
+    assert meta.cfg.make_dict() == {'val1.val2': ['val1'], 'val1.val1': ['val1'], 'val1.val3': ['val1'], 'val': 'val1'}
+    del(meta.val)
+    meta.cfg.val1.val2 = ['val2']
+    assert meta.cfg.make_dict() == {'val1.val2': ['val2'], 'val1.val1': ['val'], 'val1.val3': ['val'], 'val': 'val'}
+    del(meta.cfg.val1.val2)
+    assert meta.cfg.make_dict() == {'val1.val2': ['val'], 'val1.val1': ['val'], 'val1.val3': ['val'], 'val': 'val'}
+    meta.val1.val2 = ['val2']
+    assert meta.cfg.make_dict() == {'val1.val2': ['val2'], 'val1.val1': ['val'], 'val1.val3': ['val'], 'val': 'val'}
+    meta.cfg.val1.val3 = ['val6']
+    assert meta.cfg.make_dict() == {'val1.val2': ['val2'], 'val1.val1': ['val'], 'val1.val3': ['val6'], 'val': 'val'}
+    del(meta.val1.val2)
+    del(meta.cfg.val1.val3)
+    meta.cfg.val1.val1 = ['val3']
+    assert meta.cfg.make_dict() == {'val1.val2': ['val3'], 'val1.val1': ['val3'], 'val1.val3': ['val3'], 'val': 'val'}
+    del(meta.cfg.val1.val1)
+    assert meta.cfg.make_dict() == {'val1.val2': ['val'], 'val1.val1': ['val'], 'val1.val3': ['val'], 'val': 'val'}
+    meta.val1.val1 = ['val3']
+    assert meta.cfg.make_dict() == {'val1.val2': ['val3'], 'val1.val1': ['val3'], 'val1.val3': ['val3'], 'val': 'val'}
+    meta.cfg.val1.val2 = ['val2']
+    assert meta.cfg.make_dict() == {'val1.val2': ['val2'], 'val1.val1': ['val3'], 'val1.val3': ['val3'], 'val': 'val'}
+    meta.cfg.val1.val1.append('rah')
+    assert meta.cfg.make_dict() == {'val1.val2': ['val2', 'rah'], 'val1.val1': ['val3', 'rah'], 'val1.val3': ['val3', 'rah'], 'val': 'val'}
+    meta.val1.val1 = ['val4']
+    assert meta.cfg.make_dict() == {'val1.val2': ['val2', 'rah'], 'val1.val1': ['val3', 'rah'], 'val1.val3': ['val3', 'rah'], 'val': 'val'}

tiramisu/config.py

@@ -290,7 +290,7 @@ class SubConfig(object):
                      not_raises=not_raises)
 
     def setattr(self, name, value, force_permissive=False, not_raises=False, index=None,
-                _setting_properties=undefined):
+                _setting_properties=undefined, _commit=True):
         if name.startswith('_impl_'):
             return object.__setattr__(self, name, value)
         context = self._cfgimpl_get_context()
@@ -302,7 +302,8 @@ class SubConfig(object):
                                                              _setting_properties=_setting_properties)
             return homeconfig.setattr(name, value, force_permissive,
                                       not_raises, index=index,
-                                      _setting_properties=_setting_properties)
+                                      _setting_properties=_setting_properties,
+                                      _commit=_commit)
         child = self.cfgimpl_get_description().__getattr__(name,
                                                            context=context)
         if isinstance(child, OptionDescription) or isinstance(child, SynDynOptionDescription):
@@ -312,13 +313,15 @@ class SubConfig(object):
             path = context.cfgimpl_get_description().impl_get_path_by_opt(
                 child._impl_getopt())
             context.setattr(path, value, force_permissive, not_raises, index=index,
-                            _setting_properties=_setting_properties)
+                            _setting_properties=_setting_properties,
+                            _commit=_commit)
         else:
             subpath = self._get_subpath(name)
             self.cfgimpl_get_values().setitem(child, value, subpath,
                                               force_permissive=force_permissive,
                                               not_raises=not_raises, index=index,
-                                              _setting_properties=_setting_properties)
+                                              _setting_properties=_setting_properties,
+                                              _commit=_commit)
 
     def __delattr__(self, name):
         context = self._cfgimpl_get_context()
@@ -871,16 +874,19 @@ class GroupConfig(_CommonConfig):
         for child in self._impl_children:
             child.cfgimpl_reset_cache(only_expired=only_expired, only=only, opt=opt, path=path)
 
-    def set_value(self, path, value):
+    def set_value(self, path, value, _commit=True):
         """Setattr not in current GroupConfig, but in each children
         """
         for child in self._impl_children:
             if isinstance(child, MetaConfig):
-                child.set_value(path, value, only_config=True)
+                child.set_value(path, value, only_config=True, _commit=False)
             elif isinstance(child, GroupConfig):
-                child.set_value(path, value)
+                child.set_value(path, value, _commit=False)
             else:
-                child.setattr(path, value, not_raises=True)
+                child.setattr(path, value, not_raises=True, _commit=False)
+        if _commit:
+            self.cfgimpl_get_values()._p_.commit()
+
 
     def find_firsts(self, byname=None, bypath=undefined, byoption=undefined,
                     byvalue=undefined, raise_if_not_found=True, _sub=False,
@@ -975,7 +981,7 @@ class MetaConfig(GroupConfig):
 
     def set_value(self, path, value, force_default=False,
                   force_dont_change_value=False, force_default_if_same=False,
-                  only_config=False):
+                  only_config=False, _commit=True):
         """only_config: could be set if you want modify value in all Config included in
                         this MetaConfig
         """
@@ -984,12 +990,13 @@ class MetaConfig(GroupConfig):
                 raise ValueError(_('force_default, force_default_if_same or '
                                    'force_dont_change_value cannot be set with'
                                    ' only_config'))
-            return super(MetaConfig, self).set_value(path, value)
+            return super(MetaConfig, self).set_value(path, value, _commit=_commit)
         if force_default or force_default_if_same or force_dont_change_value:
             if force_default and force_dont_change_value:
                 raise ValueError(_('force_default and force_dont_change_value'
                                    ' cannot be set together'))
             opt = self.cfgimpl_get_description().impl_get_opt_by_path(path)
+            setting_properties = self.cfgimpl_get_settings()._getproperties(read_write=False)
             for child in self._impl_children:
                 if force_default_if_same or force_default:
                     if force_default_if_same:
@@ -999,14 +1006,16 @@ class MetaConfig(GroupConfig):
                             child_value = child.getattr(path)
                     if force_default or value == child_value:
                         child.cfgimpl_get_values().reset(opt, path=path,
-                                                         validate=False)
+                                                         validate=False,
+                                                         _setting_properties=setting_properties,
+                                                         _commit=False)
                         continue
                 if force_dont_change_value:
-                    child_value = child.getattr(path)
+                    child_value = child.getattr(path, _setting_properties=setting_properties)
                     if value != child_value:
-                        setattr(child, path, child_value)
+                        child.setattr(path, child_value, _commit=False)
 
-        setattr(self, path, value)
+        self.setattr(path, value, _commit=_commit)
 
     def new_config(self, session_id=None, persistent=False, name=undefined):
         return Config(self._impl_descr, _duplicate=True, session_id=session_id, name=name,

tiramisu/option/masterslave.py

@@ -99,9 +99,10 @@ class MasterSlaves(object):
         else:
             return opt == self._p_._sm_getmaster() or opt in self._p_._sm_getslaves()
 
-    def reset(self, opt, values, setting_properties):
+    def reset(self, opt, values, setting_properties, _commit=True):
         for slave in self.getslaves(opt):
-            values.reset(slave, validate=False, _setting_properties=setting_properties)
+            values.reset(slave, validate=False, _setting_properties=setting_properties,
+                         _commit=_commit)
 
     def pop(self, opt, values, index):
         for slave in self.getslaves(opt):

tiramisu/option/optiondescription.py

@@ -204,6 +204,7 @@ class OptionDescription(BaseOption, StorageOptionDescription):
 
     def impl_build_force_store_values(self, config, force_store_values):
         session = config._impl_values._p_.getsession()
+        value_set = False
         for subpath, option in self._cache_force_store_values:
             if option.impl_is_master_slaves('slave'):
                 # problem with index
@@ -218,8 +219,12 @@ class OptionDescription(BaseOption, StorageOptionDescription):
                                                                       validate=False,
                                                                       trusted_cached_properties=False,
                                                                       validate_properties=True)
+                value_set = True
                 config._impl_values._p_.setvalue(subpath, value,
-                                                 owners.forced, None, session)
+                                                 owners.forced, None, session, False)
+
+        if value_set:
+            config._impl_values._p_.commit()
 
     # ____________________________________________________________
     def impl_set_group_type(self, group_type):

tiramisu/storage/dictionary/value.py

@@ -32,6 +32,9 @@ class Values(Cache):
         # should init cache too
         super(Values, self).__init__(storage)
 
+    def commit(self):
+        pass
+
     def getsession(self):
         pass
 
@@ -63,7 +66,7 @@ class Values(Cache):
         values.append(tuple(lst))
         return vidx
     # value
-    def setvalue(self, path, value, owner, index, session):
+    def setvalue(self, path, value, owner, index, session, commit):
         """set value for a path
         a specified value must be associated to an owner
         """
@@ -94,7 +97,7 @@ class Values(Cache):
         """
         return path in self._values[0]
 
-    def resetvalue(self, path, session):
+    def resetvalue(self, path, session, commit):
         """remove value means delete value in storage
         """
         def _resetvalue(nb):

tiramisu/storage/sqlite3/storage.py

@@ -21,8 +21,6 @@ from os.path import basename, splitext, join, isfile
 import sqlite3
 from glob import glob
 from ..util import SerializeObject
-global idx
-idx = 0
 
 
 class Setting(SerializeObject):
@@ -94,15 +92,17 @@ class Storage(object):
             self.execute(settings_table, commit=False)
             self.execute(permissives_table)
 
+    def commit(self):
+        #print('ca commit')
+        self._conn.commit()
+
     def execute(self, sql, params=None, commit=True):
-        global idx
+        #print(sql, params)
-        idx += 1
-        print(idx, sql, params)
         if params is None:
             params = tuple()
         self._cursor.execute(sql, params)
         if commit:
-            self._conn.commit()
+            self.commit()
 
     def select(self, sql, params=None, only_one=True):
         self.execute(sql, params=params, commit=False)

tiramisu/storage/sqlite3/value.py

@@ -43,8 +43,11 @@ class Values(Sqlite3DB):
         request += "LIMIT 1"
         return self._storage.select(request, params)
 
+    def commit(self):
+        self._storage.commit()
+
     # value
-    def setvalue(self, path, value, owner, index, session):
+    def setvalue(self, path, value, owner, index, session, commit):
         """set value for an option
         a specified value must be associated to an owner
         """
@@ -58,7 +61,7 @@ class Values(Sqlite3DB):
                                                       str(owner),
                                                       index,
                                                       self._session_id),
-                                  commit=True)
+                                  commit=commit)
         else:
             self._storage.execute("DELETE FROM value WHERE path = ? AND session_id = ?",
                                   (path, self._session_id),
@@ -67,7 +70,7 @@ class Values(Sqlite3DB):
                                   "(?, ?, ?, ?)", (path, self._sqlite_encode(value),
                                                    str(owner),
                                                    self._session_id),
-                                  commit=True)
+                                  commit=commit)
 
     def getvalue(self, path, session, index=None):
         """get value for an option
@@ -86,11 +89,12 @@ class Values(Sqlite3DB):
         path = self._sqlite_encode_path(path)
         return self._sqlite_select(path, index) is not None
 
-    def resetvalue(self, path, session):
+    def resetvalue(self, path, session, _commit):
         """remove value means delete value in storage
         """
         path = self._sqlite_encode_path(path)
-        self._storage.execute("DELETE FROM value WHERE path = ? AND session_id = ?", (path, self._session_id))
+        self._storage.execute("DELETE FROM value WHERE path = ? AND session_id = ?", (path, self._session_id),
+                commit=_commit)
 
     def get_modified_values(self):
         """return all values in a dictionary

tiramisu/value.py

@@ -58,30 +58,16 @@ class Values(object):
     def _get_multi(self, opt, path):
         return Multi([], self.context, opt, path)
 
-    def _getdefaultvalue(self, opt, path, with_meta, index, submulti_index, validate):
+    def _getdefaultvalue(self, opt, path, with_meta, index, submulti_index, validate,
-        # if value has callback and is not set
+                         _orig_context=undefined):
-        if opt.impl_has_callback():
+        if _orig_context is undefined:
-            callback, callback_params = opt.impl_get_callback()
+            _orig_context = self._getcontext()
-            value = carry_out_calculation(opt, context=self._getcontext(),
-                                          callback=callback,
-                                          callback_params=callback_params,
-                                          index=index, validate=validate)
-            if isinstance(value, list) and index is not None:
-                #if return a list and index is set, return value only if
-                #it's a submulti without submulti_index and without list of list
-                if opt.impl_is_submulti() and submulti_index is undefined and \
-                        (len(value) == 0 or not isinstance(value[0], list)):
-                    return value
-                if not opt.impl_is_submulti() and len(value) > index:
-                    return value[index]
-            else:
-                return value
         if with_meta:
             meta = self._getcontext().cfgimpl_get_meta()
             if meta is not None:
                 value = meta.cfgimpl_get_values(
                 )._get_cached_value(opt, path, index=index, submulti_index=submulti_index,
-                                    from_masterslave=True)
+                                    from_masterslave=True, _orig_context=_orig_context)
                 if isinstance(value, Exception):
                     if not isinstance(value, PropertiesOptionError):  # pragma: no cover
                         raise value
@@ -103,6 +89,23 @@ class Values(object):
                             value = new_value
                             del new_value
                     return value
+        # if value has callback and is not set
+        if opt.impl_has_callback():
+            callback, callback_params = opt.impl_get_callback()
+            value = carry_out_calculation(opt, context=_orig_context,
+                                          callback=callback,
+                                          callback_params=callback_params,
+                                          index=index, validate=validate)
+            if isinstance(value, list) and index is not None:
+                #if return a list and index is set, return value only if
+                #it's a submulti without submulti_index and without list of list
+                if opt.impl_is_submulti() and submulti_index is undefined and \
+                        (len(value) == 0 or not isinstance(value[0], list)):
+                    return value
+                if not opt.impl_is_submulti() and len(value) > index:
+                    return value[index]
+            else:
+                return value
         # now try to get default value
         value = opt.impl_getdefault()
         if opt.impl_is_multi() and index is not None:
@@ -116,7 +119,7 @@ class Values(object):
         return value
 
     def _getvalue(self, opt, path, self_properties, index, submulti_index,
-                  with_meta, masterlen, session, validate):
+                  with_meta, masterlen, session, validate, _orig_context):
         """actually retrieves the value
 
         :param opt: the `option.Option()` object
@@ -141,7 +144,7 @@ class Values(object):
             else:
                 return value
         return self._getdefaultvalue(opt, path, with_meta, index,
-                                     submulti_index, validate)
+                                     submulti_index, validate, _orig_context)
 
     def get_modified_values(self):
         return self._p_.get_modified_values()
@@ -165,7 +168,7 @@ class Values(object):
         """overrides the builtins `del()` instructions"""
         self.reset(opt)
 
-    def reset(self, opt, path=None, validate=True, _setting_properties=None):
+    def reset(self, opt, path=None, validate=True, _setting_properties=None, _commit=True):
         context = self._getcontext()
         setting = context.cfgimpl_get_settings()
         if path is None:
@@ -186,7 +189,7 @@ class Values(object):
             if isinstance(ret, Exception):
                 raise ret
         if opt.impl_is_master_slaves('master'):
-            opt.impl_get_master_slaves().reset(opt, self, _setting_properties)
+            opt.impl_get_master_slaves().reset(opt, self, _setting_properties, _commit=_commit)
         if hasvalue:
             if 'force_store_value' in setting._getproperties(opt=opt,
                                                              path=path,
@@ -196,9 +199,9 @@ class Values(object):
                 value = self._getdefaultvalue(opt, path, True, undefined, undefined, validate)
                 if isinstance(value, Exception):  # pragma: no cover
                     raise value
-                self._setvalue(opt, path, value, force_owner=owners.forced)
+                self._setvalue(opt, path, value, force_owner=owners.forced, commit=_commit)
             else:
-                self._p_.resetvalue(path, session)
+                self._p_.resetvalue(path, session, _commit)
             context.cfgimpl_reset_cache(opt=opt, path=path, only=('values', 'properties'))
 
     def _isempty(self, opt, value, force_allow_empty_list=False, index=None):
@@ -239,7 +242,7 @@ class Values(object):
                           setting_properties=undefined, self_properties=undefined,
                           index=None, submulti_index=undefined, from_masterslave=False,
                           with_meta=True, masterlen=undefined, check_frozen=False,
-                          session=None, display_warnings=True):
+                          session=None, display_warnings=True, _orig_context=undefined):
         context = self._getcontext()
         settings = context.cfgimpl_get_settings()
         if path is None:
@@ -252,7 +255,8 @@ class Values(object):
                                                       read_write=False,
                                                       setting_properties=setting_properties,
                                                       index=index)
-        if 'cache' in setting_properties and self._p_.hascache(path, index):
+        if 'cache' in setting_properties and self._p_.hascache(path, index) and \
+                _orig_context is undefined:
             if 'expire' in setting_properties:
                 ntime = int(time())
             is_cached, value = self._p_.getcache(path, ntime, index)
@@ -295,13 +299,14 @@ class Values(object):
                                             submulti_index=submulti_index,
                                             check_frozen=check_frozen,
                                             session=session,
-                                            display_warnings=display_warnings)
+                                            display_warnings=display_warnings,
+                                            _orig_context=_orig_context)
         if isinstance(val, Exception):
             return val
         # cache doesn't work with SubMulti yet
         if not isinstance(val, SubMulti) and 'cache' in setting_properties and \
                 validate and validate_properties and force_permissive is False \
-                and trusted_cached_properties is True:
+                and trusted_cached_properties is True and _orig_context is undefined:
             if 'expire' in setting_properties:
                 if ntime is None:
                     ntime = int(time())
@@ -316,7 +321,8 @@ class Values(object):
                              with_meta=True,
                              masterlen=undefined,
                              check_frozen=False,
-                             session=None, display_warnings=True):
+                             session=None, display_warnings=True,
+                             _orig_context=undefined):
         """same has getitem but don't touch the cache
         index is None for slave value, if value returned is not a list, just return []
         """
@@ -326,7 +332,7 @@ class Values(object):
         if session is None:
             session = self._p_.getsession()
         value = self._getvalue(opt, path, self_properties, index, submulti_index,
-                               with_meta, masterlen, session, validate)
+                               with_meta, masterlen, session, validate, _orig_context)
         if isinstance(value, Exception):
             value_error = True
             if isinstance(value, ConfigError):
@@ -400,7 +406,7 @@ class Values(object):
 
     def setitem(self, opt, value, path, force_permissive=False,
                 check_frozen=True, not_raises=False, index=None,
-                _setting_properties=undefined):
+                _setting_properties=undefined, _commit=True):
         # check_frozen is, for example, used with "force_store_value"
         # user didn't change value, so not write
         # valid opt
@@ -430,9 +436,9 @@ class Values(object):
                 raise err
             opt.impl_validate(value, fake_context, display_error=False,
                               setting_properties=_setting_properties)
-        self._setvalue(opt, path, value, index=index)
+        self._setvalue(opt, path, value, index=index, commit=_commit)
 
-    def _setvalue(self, opt, path, value, force_owner=undefined, index=None):
+    def _setvalue(self, opt, path, value, force_owner=undefined, index=None, commit=True):
         context = self._getcontext()
         context.cfgimpl_reset_cache(opt=opt, path=path, only=('values', 'properties'))
         if force_owner is undefined:
@@ -453,13 +459,13 @@ class Values(object):
         #FIXME pourquoi là et pas dans masterslaves ??
         if opt.impl_is_master_slaves('slave'):
             if index is not None:
-                self._p_.setvalue(path, value, owner, index, session)
+                self._p_.setvalue(path, value, owner, index, session, commit)
             else:
-                self._p_.resetvalue(path, session)
+                self._p_.resetvalue(path, session, commit)
                 for idx, val in enumerate(value):
-                    self._p_.setvalue(path, val, owner, idx, session)
+                    self._p_.setvalue(path, val, owner, idx, session, commit)
         else:
-            self._p_.setvalue(path, value, owner, None, session)
+            self._p_.setvalue(path, value, owner, None, session, commit)
         del(session)
 
     def validate(self, opt, value, path, check_frozen=True, force_permissive=False,