less commit + better meta support

This commit is contained in:
Emmanuel Garette 2017-07-16 23:11:12 +02:00
parent 7fb0ee2575
commit 90a2072fc5
8 changed files with 167 additions and 68 deletions

View file

@ -11,6 +11,10 @@ from tiramisu.error import ConfigError, ConflictError
owners.addowner('meta')
def return_value(value=None):
return value
def raise_exception():
raise Exception('test')
@ -528,3 +532,70 @@ def test_meta_exception_meta():
meta = MetaConfig([conf1, conf2])
meta.read_write()
raises(Exception, "conf1.make_dict()")
def test_meta_callback():
val1 = StrOption('val1', "", 'val')
val2 = StrOption('val2', "", callback=return_value, callback_params={'': ((val1, False),)})
val3 = StrOption('val3', "", callback=return_value, callback_params={'': ('yes',)})
val4 = StrOption('val4', "", callback=return_value, callback_params={'value': ((val1, False),)})
val5 = StrOption('val5', "", callback=return_value, callback_params={'value': ('yes',)})
maconfig = OptionDescription('rootconfig', '', [val1, val2, val3, val4, val5])
cfg = Config(maconfig, name='cfg')
meta = MetaConfig([cfg])
meta.read_write()
assert meta.cfg.make_dict() == {'val3': 'yes', 'val2': 'val', 'val1': 'val', 'val5': 'yes', 'val4': 'val'}
meta.cfg.val1 = 'new'
assert meta.cfg.make_dict() == {'val3': 'yes', 'val2': 'new', 'val1': 'new', 'val5': 'yes', 'val4': 'new'}
del(meta.cfg.val1)
meta.val1 = 'new'
assert meta.cfg.make_dict() == {'val3': 'yes', 'val2': 'new', 'val1': 'new', 'val5': 'yes', 'val4': 'new'}
#del(meta.val1)
meta.cfg.val4 = 'new1'
assert meta.cfg.make_dict() == {'val3': 'yes', 'val2': 'new', 'val1': 'new', 'val5': 'yes', 'val4': 'new1'}
del(meta.cfg.val4)
meta.val4 = 'new1'
assert meta.cfg.make_dict() == {'val3': 'yes', 'val2': 'new', 'val1': 'new', 'val5': 'yes', 'val4': 'new1'}
del(meta.val4)
def test_meta_callback_slave():
val = StrOption('val', "", default='val')
val1 = StrOption('val1', "", multi=True, callback=return_value, callback_params={'': ((val, False),)})
val3 = StrOption('val2', "", multi=True, callback=return_value, callback_params={'': ((val1, False),)})
val4 = StrOption('val3', "", multi=True, callback=return_value, callback_params={'': ((val1, False),)})
interface1 = OptionDescription('val1', '', [val1, val3, val4])
interface1.impl_set_group_type(groups.master)
maconfig = OptionDescription('rootconfig', '', [val, interface1])
cfg = Config(maconfig, name='cfg')
meta = MetaConfig([cfg])
meta.read_write()
assert meta.cfg.make_dict() == {'val1.val2': ['val'], 'val1.val1': ['val'], 'val1.val3': ['val'], 'val': 'val'}
meta.cfg.val = 'val1'
assert meta.cfg.make_dict() == {'val1.val2': ['val1'], 'val1.val1': ['val1'], 'val1.val3': ['val1'], 'val': 'val1'}
del(meta.cfg.val)
meta.val = 'val1'
assert meta.cfg.make_dict() == {'val1.val2': ['val1'], 'val1.val1': ['val1'], 'val1.val3': ['val1'], 'val': 'val1'}
del(meta.val)
meta.cfg.val1.val2 = ['val2']
assert meta.cfg.make_dict() == {'val1.val2': ['val2'], 'val1.val1': ['val'], 'val1.val3': ['val'], 'val': 'val'}
del(meta.cfg.val1.val2)
assert meta.cfg.make_dict() == {'val1.val2': ['val'], 'val1.val1': ['val'], 'val1.val3': ['val'], 'val': 'val'}
meta.val1.val2 = ['val2']
assert meta.cfg.make_dict() == {'val1.val2': ['val2'], 'val1.val1': ['val'], 'val1.val3': ['val'], 'val': 'val'}
meta.cfg.val1.val3 = ['val6']
assert meta.cfg.make_dict() == {'val1.val2': ['val2'], 'val1.val1': ['val'], 'val1.val3': ['val6'], 'val': 'val'}
del(meta.val1.val2)
del(meta.cfg.val1.val3)
meta.cfg.val1.val1 = ['val3']
assert meta.cfg.make_dict() == {'val1.val2': ['val3'], 'val1.val1': ['val3'], 'val1.val3': ['val3'], 'val': 'val'}
del(meta.cfg.val1.val1)
assert meta.cfg.make_dict() == {'val1.val2': ['val'], 'val1.val1': ['val'], 'val1.val3': ['val'], 'val': 'val'}
meta.val1.val1 = ['val3']
assert meta.cfg.make_dict() == {'val1.val2': ['val3'], 'val1.val1': ['val3'], 'val1.val3': ['val3'], 'val': 'val'}
meta.cfg.val1.val2 = ['val2']
assert meta.cfg.make_dict() == {'val1.val2': ['val2'], 'val1.val1': ['val3'], 'val1.val3': ['val3'], 'val': 'val'}
meta.cfg.val1.val1.append('rah')
assert meta.cfg.make_dict() == {'val1.val2': ['val2', 'rah'], 'val1.val1': ['val3', 'rah'], 'val1.val3': ['val3', 'rah'], 'val': 'val'}
meta.val1.val1 = ['val4']
assert meta.cfg.make_dict() == {'val1.val2': ['val2', 'rah'], 'val1.val1': ['val3', 'rah'], 'val1.val3': ['val3', 'rah'], 'val': 'val'}

View file

@ -290,7 +290,7 @@ class SubConfig(object):
not_raises=not_raises)
def setattr(self, name, value, force_permissive=False, not_raises=False, index=None,
_setting_properties=undefined):
_setting_properties=undefined, _commit=True):
if name.startswith('_impl_'):
return object.__setattr__(self, name, value)
context = self._cfgimpl_get_context()
@ -302,7 +302,8 @@ class SubConfig(object):
_setting_properties=_setting_properties)
return homeconfig.setattr(name, value, force_permissive,
not_raises, index=index,
_setting_properties=_setting_properties)
_setting_properties=_setting_properties,
_commit=_commit)
child = self.cfgimpl_get_description().__getattr__(name,
context=context)
if isinstance(child, OptionDescription) or isinstance(child, SynDynOptionDescription):
@ -312,13 +313,15 @@ class SubConfig(object):
path = context.cfgimpl_get_description().impl_get_path_by_opt(
child._impl_getopt())
context.setattr(path, value, force_permissive, not_raises, index=index,
_setting_properties=_setting_properties)
_setting_properties=_setting_properties,
_commit=_commit)
else:
subpath = self._get_subpath(name)
self.cfgimpl_get_values().setitem(child, value, subpath,
force_permissive=force_permissive,
not_raises=not_raises, index=index,
_setting_properties=_setting_properties)
_setting_properties=_setting_properties,
_commit=_commit)
def __delattr__(self, name):
context = self._cfgimpl_get_context()
@ -871,16 +874,19 @@ class GroupConfig(_CommonConfig):
for child in self._impl_children:
child.cfgimpl_reset_cache(only_expired=only_expired, only=only, opt=opt, path=path)
def set_value(self, path, value):
def set_value(self, path, value, _commit=True):
"""Setattr not in current GroupConfig, but in each children
"""
for child in self._impl_children:
if isinstance(child, MetaConfig):
child.set_value(path, value, only_config=True)
child.set_value(path, value, only_config=True, _commit=False)
elif isinstance(child, GroupConfig):
child.set_value(path, value)
child.set_value(path, value, _commit=False)
else:
child.setattr(path, value, not_raises=True)
child.setattr(path, value, not_raises=True, _commit=False)
if _commit:
self.cfgimpl_get_values()._p_.commit()
def find_firsts(self, byname=None, bypath=undefined, byoption=undefined,
byvalue=undefined, raise_if_not_found=True, _sub=False,
@ -975,7 +981,7 @@ class MetaConfig(GroupConfig):
def set_value(self, path, value, force_default=False,
force_dont_change_value=False, force_default_if_same=False,
only_config=False):
only_config=False, _commit=True):
"""only_config: could be set if you want modify value in all Config included in
this MetaConfig
"""
@ -984,12 +990,13 @@ class MetaConfig(GroupConfig):
raise ValueError(_('force_default, force_default_if_same or '
'force_dont_change_value cannot be set with'
' only_config'))
return super(MetaConfig, self).set_value(path, value)
return super(MetaConfig, self).set_value(path, value, _commit=_commit)
if force_default or force_default_if_same or force_dont_change_value:
if force_default and force_dont_change_value:
raise ValueError(_('force_default and force_dont_change_value'
' cannot be set together'))
opt = self.cfgimpl_get_description().impl_get_opt_by_path(path)
setting_properties = self.cfgimpl_get_settings()._getproperties(read_write=False)
for child in self._impl_children:
if force_default_if_same or force_default:
if force_default_if_same:
@ -999,14 +1006,16 @@ class MetaConfig(GroupConfig):
child_value = child.getattr(path)
if force_default or value == child_value:
child.cfgimpl_get_values().reset(opt, path=path,
validate=False)
validate=False,
_setting_properties=setting_properties,
_commit=False)
continue
if force_dont_change_value:
child_value = child.getattr(path)
child_value = child.getattr(path, _setting_properties=setting_properties)
if value != child_value:
setattr(child, path, child_value)
child.setattr(path, child_value, _commit=False)
setattr(self, path, value)
self.setattr(path, value, _commit=_commit)
def new_config(self, session_id=None, persistent=False, name=undefined):
return Config(self._impl_descr, _duplicate=True, session_id=session_id, name=name,

View file

@ -99,9 +99,10 @@ class MasterSlaves(object):
else:
return opt == self._p_._sm_getmaster() or opt in self._p_._sm_getslaves()
def reset(self, opt, values, setting_properties):
def reset(self, opt, values, setting_properties, _commit=True):
for slave in self.getslaves(opt):
values.reset(slave, validate=False, _setting_properties=setting_properties)
values.reset(slave, validate=False, _setting_properties=setting_properties,
_commit=_commit)
def pop(self, opt, values, index):
for slave in self.getslaves(opt):

View file

@ -204,6 +204,7 @@ class OptionDescription(BaseOption, StorageOptionDescription):
def impl_build_force_store_values(self, config, force_store_values):
session = config._impl_values._p_.getsession()
value_set = False
for subpath, option in self._cache_force_store_values:
if option.impl_is_master_slaves('slave'):
# problem with index
@ -218,8 +219,12 @@ class OptionDescription(BaseOption, StorageOptionDescription):
validate=False,
trusted_cached_properties=False,
validate_properties=True)
value_set = True
config._impl_values._p_.setvalue(subpath, value,
owners.forced, None, session)
owners.forced, None, session, False)
if value_set:
config._impl_values._p_.commit()
# ____________________________________________________________
def impl_set_group_type(self, group_type):

View file

@ -32,6 +32,9 @@ class Values(Cache):
# should init cache too
super(Values, self).__init__(storage)
def commit(self):
pass
def getsession(self):
pass
@ -63,7 +66,7 @@ class Values(Cache):
values.append(tuple(lst))
return vidx
# value
def setvalue(self, path, value, owner, index, session):
def setvalue(self, path, value, owner, index, session, commit):
"""set value for a path
a specified value must be associated to an owner
"""
@ -94,7 +97,7 @@ class Values(Cache):
"""
return path in self._values[0]
def resetvalue(self, path, session):
def resetvalue(self, path, session, commit):
"""remove value means delete value in storage
"""
def _resetvalue(nb):

View file

@ -21,8 +21,6 @@ from os.path import basename, splitext, join, isfile
import sqlite3
from glob import glob
from ..util import SerializeObject
global idx
idx = 0
class Setting(SerializeObject):
@ -94,15 +92,17 @@ class Storage(object):
self.execute(settings_table, commit=False)
self.execute(permissives_table)
def commit(self):
#print('ca commit')
self._conn.commit()
def execute(self, sql, params=None, commit=True):
global idx
idx += 1
print(idx, sql, params)
#print(sql, params)
if params is None:
params = tuple()
self._cursor.execute(sql, params)
if commit:
self._conn.commit()
self.commit()
def select(self, sql, params=None, only_one=True):
self.execute(sql, params=params, commit=False)

View file

@ -43,8 +43,11 @@ class Values(Sqlite3DB):
request += "LIMIT 1"
return self._storage.select(request, params)
def commit(self):
self._storage.commit()
# value
def setvalue(self, path, value, owner, index, session):
def setvalue(self, path, value, owner, index, session, commit):
"""set value for an option
a specified value must be associated to an owner
"""
@ -58,7 +61,7 @@ class Values(Sqlite3DB):
str(owner),
index,
self._session_id),
commit=True)
commit=commit)
else:
self._storage.execute("DELETE FROM value WHERE path = ? AND session_id = ?",
(path, self._session_id),
@ -67,7 +70,7 @@ class Values(Sqlite3DB):
"(?, ?, ?, ?)", (path, self._sqlite_encode(value),
str(owner),
self._session_id),
commit=True)
commit=commit)
def getvalue(self, path, session, index=None):
"""get value for an option
@ -86,11 +89,12 @@ class Values(Sqlite3DB):
path = self._sqlite_encode_path(path)
return self._sqlite_select(path, index) is not None
def resetvalue(self, path, session):
def resetvalue(self, path, session, _commit):
"""remove value means delete value in storage
"""
path = self._sqlite_encode_path(path)
self._storage.execute("DELETE FROM value WHERE path = ? AND session_id = ?", (path, self._session_id))
self._storage.execute("DELETE FROM value WHERE path = ? AND session_id = ?", (path, self._session_id),
commit=_commit)
def get_modified_values(self):
"""return all values in a dictionary

View file

@ -58,30 +58,16 @@ class Values(object):
def _get_multi(self, opt, path):
return Multi([], self.context, opt, path)
def _getdefaultvalue(self, opt, path, with_meta, index, submulti_index, validate):
# if value has callback and is not set
if opt.impl_has_callback():
callback, callback_params = opt.impl_get_callback()
value = carry_out_calculation(opt, context=self._getcontext(),
callback=callback,
callback_params=callback_params,
index=index, validate=validate)
if isinstance(value, list) and index is not None:
#if return a list and index is set, return value only if
#it's a submulti without submulti_index and without list of list
if opt.impl_is_submulti() and submulti_index is undefined and \
(len(value) == 0 or not isinstance(value[0], list)):
return value
if not opt.impl_is_submulti() and len(value) > index:
return value[index]
else:
return value
def _getdefaultvalue(self, opt, path, with_meta, index, submulti_index, validate,
_orig_context=undefined):
if _orig_context is undefined:
_orig_context = self._getcontext()
if with_meta:
meta = self._getcontext().cfgimpl_get_meta()
if meta is not None:
value = meta.cfgimpl_get_values(
)._get_cached_value(opt, path, index=index, submulti_index=submulti_index,
from_masterslave=True)
from_masterslave=True, _orig_context=_orig_context)
if isinstance(value, Exception):
if not isinstance(value, PropertiesOptionError): # pragma: no cover
raise value
@ -103,6 +89,23 @@ class Values(object):
value = new_value
del new_value
return value
# if value has callback and is not set
if opt.impl_has_callback():
callback, callback_params = opt.impl_get_callback()
value = carry_out_calculation(opt, context=_orig_context,
callback=callback,
callback_params=callback_params,
index=index, validate=validate)
if isinstance(value, list) and index is not None:
#if return a list and index is set, return value only if
#it's a submulti without submulti_index and without list of list
if opt.impl_is_submulti() and submulti_index is undefined and \
(len(value) == 0 or not isinstance(value[0], list)):
return value
if not opt.impl_is_submulti() and len(value) > index:
return value[index]
else:
return value
# now try to get default value
value = opt.impl_getdefault()
if opt.impl_is_multi() and index is not None:
@ -116,7 +119,7 @@ class Values(object):
return value
def _getvalue(self, opt, path, self_properties, index, submulti_index,
with_meta, masterlen, session, validate):
with_meta, masterlen, session, validate, _orig_context):
"""actually retrieves the value
:param opt: the `option.Option()` object
@ -141,7 +144,7 @@ class Values(object):
else:
return value
return self._getdefaultvalue(opt, path, with_meta, index,
submulti_index, validate)
submulti_index, validate, _orig_context)
def get_modified_values(self):
return self._p_.get_modified_values()
@ -165,7 +168,7 @@ class Values(object):
"""overrides the builtins `del()` instructions"""
self.reset(opt)
def reset(self, opt, path=None, validate=True, _setting_properties=None):
def reset(self, opt, path=None, validate=True, _setting_properties=None, _commit=True):
context = self._getcontext()
setting = context.cfgimpl_get_settings()
if path is None:
@ -186,7 +189,7 @@ class Values(object):
if isinstance(ret, Exception):
raise ret
if opt.impl_is_master_slaves('master'):
opt.impl_get_master_slaves().reset(opt, self, _setting_properties)
opt.impl_get_master_slaves().reset(opt, self, _setting_properties, _commit=_commit)
if hasvalue:
if 'force_store_value' in setting._getproperties(opt=opt,
path=path,
@ -196,9 +199,9 @@ class Values(object):
value = self._getdefaultvalue(opt, path, True, undefined, undefined, validate)
if isinstance(value, Exception): # pragma: no cover
raise value
self._setvalue(opt, path, value, force_owner=owners.forced)
self._setvalue(opt, path, value, force_owner=owners.forced, commit=_commit)
else:
self._p_.resetvalue(path, session)
self._p_.resetvalue(path, session, _commit)
context.cfgimpl_reset_cache(opt=opt, path=path, only=('values', 'properties'))
def _isempty(self, opt, value, force_allow_empty_list=False, index=None):
@ -239,7 +242,7 @@ class Values(object):
setting_properties=undefined, self_properties=undefined,
index=None, submulti_index=undefined, from_masterslave=False,
with_meta=True, masterlen=undefined, check_frozen=False,
session=None, display_warnings=True):
session=None, display_warnings=True, _orig_context=undefined):
context = self._getcontext()
settings = context.cfgimpl_get_settings()
if path is None:
@ -252,7 +255,8 @@ class Values(object):
read_write=False,
setting_properties=setting_properties,
index=index)
if 'cache' in setting_properties and self._p_.hascache(path, index):
if 'cache' in setting_properties and self._p_.hascache(path, index) and \
_orig_context is undefined:
if 'expire' in setting_properties:
ntime = int(time())
is_cached, value = self._p_.getcache(path, ntime, index)
@ -295,13 +299,14 @@ class Values(object):
submulti_index=submulti_index,
check_frozen=check_frozen,
session=session,
display_warnings=display_warnings)
display_warnings=display_warnings,
_orig_context=_orig_context)
if isinstance(val, Exception):
return val
# cache doesn't work with SubMulti yet
if not isinstance(val, SubMulti) and 'cache' in setting_properties and \
validate and validate_properties and force_permissive is False \
and trusted_cached_properties is True:
and trusted_cached_properties is True and _orig_context is undefined:
if 'expire' in setting_properties:
if ntime is None:
ntime = int(time())
@ -316,7 +321,8 @@ class Values(object):
with_meta=True,
masterlen=undefined,
check_frozen=False,
session=None, display_warnings=True):
session=None, display_warnings=True,
_orig_context=undefined):
"""same has getitem but don't touch the cache
index is None for slave value, if value returned is not a list, just return []
"""
@ -326,7 +332,7 @@ class Values(object):
if session is None:
session = self._p_.getsession()
value = self._getvalue(opt, path, self_properties, index, submulti_index,
with_meta, masterlen, session, validate)
with_meta, masterlen, session, validate, _orig_context)
if isinstance(value, Exception):
value_error = True
if isinstance(value, ConfigError):
@ -400,7 +406,7 @@ class Values(object):
def setitem(self, opt, value, path, force_permissive=False,
check_frozen=True, not_raises=False, index=None,
_setting_properties=undefined):
_setting_properties=undefined, _commit=True):
# check_frozen is, for example, used with "force_store_value"
# user didn't change value, so not write
# valid opt
@ -430,9 +436,9 @@ class Values(object):
raise err
opt.impl_validate(value, fake_context, display_error=False,
setting_properties=_setting_properties)
self._setvalue(opt, path, value, index=index)
self._setvalue(opt, path, value, index=index, commit=_commit)
def _setvalue(self, opt, path, value, force_owner=undefined, index=None):
def _setvalue(self, opt, path, value, force_owner=undefined, index=None, commit=True):
context = self._getcontext()
context.cfgimpl_reset_cache(opt=opt, path=path, only=('values', 'properties'))
if force_owner is undefined:
@ -453,13 +459,13 @@ class Values(object):
#FIXME pourquoi là et pas dans masterslaves ??
if opt.impl_is_master_slaves('slave'):
if index is not None:
self._p_.setvalue(path, value, owner, index, session)
self._p_.setvalue(path, value, owner, index, session, commit)
else:
self._p_.resetvalue(path, session)
self._p_.resetvalue(path, session, commit)
for idx, val in enumerate(value):
self._p_.setvalue(path, val, owner, idx, session)
self._p_.setvalue(path, val, owner, idx, session, commit)
else:
self._p_.setvalue(path, value, owner, None, session)
self._p_.setvalue(path, value, owner, None, session, commit)
del(session)
def validate(self, opt, value, path, check_frozen=True, force_permissive=False,