From 4a737c5b9d239b7e824663463e15d13e7c7fdf6a Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Fri, 22 Feb 2019 07:25:57 +0100 Subject: [PATCH] remote sqlalchemy storage (unworking) and add setting to storage --- tiramisu/__init__.py | 9 +- tiramisu/storage/__init__.py | 37 ++-- tiramisu/storage/dictionary/__init__.py | 10 +- tiramisu/storage/dictionary/storage.py | 8 +- tiramisu/storage/sqlalchemy/__init__.py | 36 ---- tiramisu/storage/sqlalchemy/setting.py | 138 -------------- tiramisu/storage/sqlalchemy/storage.py | 81 --------- tiramisu/storage/sqlalchemy/util.py | 38 ---- tiramisu/storage/sqlalchemy/value.py | 228 ------------------------ tiramisu/storage/sqlite3/__init__.py | 10 +- tiramisu/storage/sqlite3/storage.py | 17 +- 11 files changed, 59 insertions(+), 553 deletions(-) delete mode 100644 tiramisu/storage/sqlalchemy/__init__.py delete mode 100644 tiramisu/storage/sqlalchemy/setting.py delete mode 100644 tiramisu/storage/sqlalchemy/storage.py delete mode 100644 tiramisu/storage/sqlalchemy/util.py delete mode 100644 tiramisu/storage/sqlalchemy/value.py diff --git a/tiramisu/__init__.py b/tiramisu/__init__.py index fa50f43..e2ff65b 100644 --- a/tiramisu/__init__.py +++ b/tiramisu/__init__.py @@ -12,12 +12,15 @@ # # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . -from .function import Params, ParamOption, ParamValue, ParamContext, tiramisu_copy +from .function import Params, ParamOption, ParamValue, ParamContext, \ + tiramisu_copy from .option import * from .error import APIError from .api import Config, MetaConfig, GroupConfig, MixConfig from .option import __all__ as all_options from .setting import owners, undefined +from .storage import default_storage_type, StorageType, list_sessions, \ + delete_session allfuncs = ['Params', @@ -30,6 +33,10 @@ allfuncs = ['Params', 'Config', 'APIError', 'undefined', + 'default_storage_type', + 'StorageType', + 'list_sessions', + 'delete_session', 'tiramisu_copy'] allfuncs.extend(all_options) del(all_options) diff --git a/tiramisu/storage/__init__.py b/tiramisu/storage/__init__.py index 6f29a51..234f379 100644 --- a/tiramisu/storage/__init__.py +++ b/tiramisu/storage/__init__.py @@ -51,22 +51,19 @@ class StorageType(object): if self.storage_type is not None: # pragma: no cover if self.storage_type == name: return - raise ConfigError(_('storage_type is already set, cannot rebind it')) + raise ConfigError(_('storage_type is already set, ' + 'cannot rebind it')) self.storage_type = name def get(self): if self.storage_type is None: self.storage_type = self.default_storage - set_to_default = True - else: - set_to_default = False if self.mod is None: - modulepath = '{0}.storage.{1}'.format(MODULE_PATH, self.storage_type) + modulepath = '{0}.storage.{1}'.format(MODULE_PATH, + self.storage_type) try: mod = __import__(modulepath) except ImportError: # pragma: no cover - import traceback - traceback.print_exc() raise SystemError(_('cannot import the storage {0}').format( self.default_storage)) for token in modulepath.split(".")[1:]: @@ -74,10 +71,13 @@ class StorageType(object): self.mod = mod return self.mod + def setting(self, **kwargs): + mod = self.get() + for key, value in kwargs.items(): + setattr(mod.SETTING, key, value) -storage_type = StorageType() -#storage_option_type = StorageType() -#storage_option_type.set(DEFAULT_STORAGE) + +default_storage_type = StorageType() memory_storage = StorageType() memory_storage.set(MEMORY_STORAGE) @@ -89,12 +89,17 @@ def gen_storage_id(session_id, return 'c' + str(id(config)) + str(int(time())) + str(randint(0, 500)) -def get_storages(context, session_id, persistent, storage): - session_id = gen_storage_id(session_id, context) +def get_storages(context, + session_id, + persistent, + storage): + session_id = gen_storage_id(session_id, + context) if storage is None: - storage = storage_type + storage = default_storage_type imp = storage.get() - imp_storage = imp.Storage(session_id, persistent) + imp_storage = imp.Storage(session_id, + persistent) properties = imp.Properties(imp_storage) permissives = imp.Permissives(imp_storage) values = imp.Values(imp_storage) @@ -118,7 +123,7 @@ def get_default_settings_storages(): def list_sessions(): """List all available session (persistent or not persistent) """ - return storage_type.get().list_sessions() + return default_storage_type.get().list_sessions() def delete_session(session_id): @@ -126,7 +131,7 @@ def delete_session(session_id): use by an other instance :params session_id: id of session to delete """ - storage_module = storage_type.get() + storage_module = default_storage_type.get() session = storage_module.storage.getsession() storage_module.value.delete_session(session_id) storage_module.storage.delete_session(session_id) diff --git a/tiramisu/storage/dictionary/__init__.py b/tiramisu/storage/dictionary/__init__.py index 27cb12e..43d53e5 100644 --- a/tiramisu/storage/dictionary/__init__.py +++ b/tiramisu/storage/dictionary/__init__.py @@ -24,6 +24,12 @@ use it. But if something goes wrong, you will lost your modifications. """ from .value import Values from .setting import Properties, Permissives -from .storage import setting, Storage, list_sessions +from .storage import SETTING, Storage, list_sessions -__all__ = ('setting', 'Values', 'Properties', 'Permissives', 'Storage', 'list_sessions') + +__all__ = ('SETTING', + 'Values', + 'Properties', + 'Permissives', + 'Storage', + 'list_sessions') diff --git a/tiramisu/storage/dictionary/storage.py b/tiramisu/storage/dictionary/storage.py index 5c9112d..b97c4e6 100644 --- a/tiramisu/storage/dictionary/storage.py +++ b/tiramisu/storage/dictionary/storage.py @@ -18,13 +18,13 @@ from ...i18n import _ from ...error import ConflictError -class Setting(object): +class Setting: """Dictionary storage has no particular setting. """ - pass + __slots__ = tuple() -setting = Setting() +SETTING = Setting() _list_sessions = [] @@ -35,7 +35,7 @@ def list_sessions(): class Storage(object): __slots__ = ('session_id', 'persistent') storage = 'dictionary' - #if object could be serializable + # if object could be serializable serializable = True def __init__(self, session_id, persistent, test=False): diff --git a/tiramisu/storage/sqlalchemy/__init__.py b/tiramisu/storage/sqlalchemy/__init__.py deleted file mode 100644 index a4b05f9..0000000 --- a/tiramisu/storage/sqlalchemy/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (C) 2014-2019 Team tiramisu (see AUTHORS for all contributors) -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Lesser General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . -# ____________________________________________________________ -"""Default plugin for storage. All informations are store in a simple -dictionary in memory. - -You cannot have persistente informations with this kind of storage. - -The advantage of this solution is that you can easily create a Config and -use it. But if something goes wrong, you will lost your modifications. -""" -from .value import Values -from .setting import Settings -from .storage import Storage, list_sessions, delete_session, storage_setting -from .util import load - - -load() - - -__all__ = (storage_setting, Values, Settings, Storage, list_sessions, delete_session, - StorageBase) -# Base, OptionDescription) diff --git a/tiramisu/storage/sqlalchemy/setting.py b/tiramisu/storage/sqlalchemy/setting.py deleted file mode 100644 index dfd974d..0000000 --- a/tiramisu/storage/sqlalchemy/setting.py +++ /dev/null @@ -1,138 +0,0 @@ -# -*- coding: utf-8 -*- -"default plugin for setting: set it in a simple dictionary" -# Copyright (C) 2014-2019 Team tiramisu (see AUTHORS for all contributors) -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Lesser General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . -# ____________________________________________________________ -from ..util import Cache -from .util import SqlAlchemyBase -import util -from sqlalchemy import Column, Integer, String, PickleType, ForeignKey -from sqlalchemy.orm import relationship -from sqlalchemy.ext.associationproxy import association_proxy -from sqlalchemy.orm.collections import attribute_mapped_collection - - -#____________________________________________________________ -# -# properties|permissives -class _Property(SqlAlchemyBase): - __tablename__ = 'property' - id = Column(Integer, primary_key=True) - setting = Column(Integer, ForeignKey('settings.id'), nullable=False) - path = Column(String) - properties = Column(PickleType) - - def __init__(self, path, properties): - self.path = path - self.properties = properties - - -class _Permissive (SqlAlchemyBase): - __tablename__ = 'permissive' - id = Column(Integer, primary_key=True) - setting = Column(Integer, ForeignKey('settings.id'), nullable=False) - path = Column(String) - permissives = Column(PickleType) - - def __init__(self, path, permissives): - self.path = path - self.permissives = permissives - - -#____________________________________________________________ -#FIXME marche pas le cache ... de toute facon je vais faire un storage separe ! -class Settings(Cache, SqlAlchemyBase): - __tablename__ = 'settings' - id = Column(Integer, primary_key=True) - session_id = Column(String, index=True) - _props = relationship("_Property", - collection_class=attribute_mapped_collection('path'), - cascade="all, delete-orphan") - _properties = association_proxy("_props", "properties") - _perms = relationship("_Permissive", - collection_class=attribute_mapped_collection('path'), - cascade="all, delete-orphan") - _permissives = association_proxy("_perms", "permissives") - - def __init__(self, session_id, storage): - session = self.getsession() - self.session_id = session_id - super(Settings, self).__init__(storage) - session.commit() - - def getsession(self): - return util.Session() - - # properties - def setproperties(self, path, properties): - session = self.getsession() - self._properties[path] = properties - session.commit() - del(session) - - def getproperties(self, path, default_properties): - ret = self._properties.get(path) - if ret is None: - return set(default_properties) - return ret - - def hasproperties(self, path): - return path in self._properties - - def reset_all_properties(self): - session = self.getsession() - self._properties.clear() - session.commit() - del(session) - - def delproperties(self, path): - try: - session = self.getsession() - del(self._properties[path]) - session.commit() - del(session) - except KeyError: - pass - - # permissive - def setpermissive(self, path, permissive): - session = self.getsession() - self._permissives[path] = frozenset(permissive) - session.commit() - - def getpermissive(self, path=None): - ret = self._permissives.get(path, frozenset()) - #replace None by a frozenset() - return {None: frozenset()}.get(ret, ret) - - def exportation(self): - """return all modified settings in a dictionary - example: {'path1': set(['prop1', 'prop2'])} - """ - return self._properties - - def importation(self): - """return all modified permissives in a dictionary - example: {'path1': set(['perm1', 'perm2'])} - """ - return self._permissives - - -def delete_session(session_id, session): - settings_id = session.query(Settings).filter_by(session_id=session_id).first().id - for val in session.query(_Property).filter_by(settings=settings_id).all(): - session.delete(val) - for val in session.query(_Permissive).filter_by(settings=settings_id).all(): - session.delete(val) diff --git a/tiramisu/storage/sqlalchemy/storage.py b/tiramisu/storage/sqlalchemy/storage.py deleted file mode 100644 index 7ffa5e3..0000000 --- a/tiramisu/storage/sqlalchemy/storage.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (C) 2013-2019 Team tiramisu (see AUTHORS for all contributors) -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Lesser General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . -# ____________________________________________________________ -from tiramisu.i18n import _ -from .util import SqlAlchemyBase -import util -from sqlalchemy import Column, Integer, String - - -class Setting(object): - """:param extension: database file extension (by default: db) - :param dir_database: root database directory (by default: /tmp) - """ - #FIXME - extension = 'db' - dir_database = '/tmp' - - -storage_setting = Setting() - - -class Session(SqlAlchemyBase): - __tablename__ = 'session' - id = Column(Integer, primary_key=True) - session = Column(String, index=True) - - def __init__(self, session_id): - self.session = session_id - - -def list_sessions(): - session = util.Session() - ret = [] - for val in session.query(Session).all(): - ret.append(val.session) - del(session) - return ret - - -def delete_session(session_id, session): - session.delete(session.query(Session).filter_by(session=session_id).first()) - session.commit() - - -def getsession(): - return util.Session() - - -class Storage(object): - __slots__ = ('session_id', 'persistent') - storage = 'sqlalchemy' - #if object could be serializable - serializable = True - - def __init__(self, session_id, persistent, test=False): - session = getsession() - self.session_id = session_id - self.persistent = persistent - if not session.query(Session).filter_by(session=session_id).first(): - session.add(Session(session_id)) - session.commit() - del(session) - - def __del__(self): - if not self.persistent: - session = getsession() - delete_session(self.session_id, session) - del(session) diff --git a/tiramisu/storage/sqlalchemy/util.py b/tiramisu/storage/sqlalchemy/util.py deleted file mode 100644 index 2f5173f..0000000 --- a/tiramisu/storage/sqlalchemy/util.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- -"" -# Copyright (C) 2014-2019 Team tiramisu (see AUTHORS for all contributors) -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# ____________________________________________________________ -from sqlalchemy.orm import sessionmaker -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy import create_engine - - -engine = create_engine('sqlite:///:memory:') -SqlAlchemyBase = declarative_base() - -global session, Session -Session = None - - -def load(): - global session, Session - if Session is None: - #engine.echo = True - #print SqlAlchemyBase.metadata.tables.keys() - SqlAlchemyBase.metadata.create_all(engine) - Session = sessionmaker(bind=engine, expire_on_commit=False) diff --git a/tiramisu/storage/sqlalchemy/value.py b/tiramisu/storage/sqlalchemy/value.py deleted file mode 100644 index 3b778aa..0000000 --- a/tiramisu/storage/sqlalchemy/value.py +++ /dev/null @@ -1,228 +0,0 @@ -# -*- coding: utf-8 -*- -"plugin for value: set it in sqlalchemy" -# Copyright (C) 2013-2019 Team tiramisu (see AUTHORS for all contributors) -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Lesser General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program. If not, see . -# ____________________________________________________________ - - -from ..util import Cache -from .util import SqlAlchemyBase -import util -from ...setting import undefined -from sqlalchemy import Column, Integer, String, PickleType -from sqlalchemy import func -from tiramisu.setting import owners - - -#____________________________________________________________ -# -# information -class _Vinformation(SqlAlchemyBase): - __tablename__ = 'vinformation' - id = Column(Integer, primary_key=True) - session_id = Column(String, index=True) - path = Column(String, index=True) - key = Column(String) - value = Column(PickleType) - - def __init__(self, session_id, key, value): - self.session_id = session_id - self.key = key - self.value = value - - -class Value(SqlAlchemyBase): - __tablename__ = 'value' - id = Column(Integer, primary_key=True) - session_id = Column(String, index=True) - path = Column(String, index=True) - key = Column(String) - value = Column(PickleType) - owner = Column(String) - indx = Column(Integer, index=True) - - def __init__(self, session_id, path, value, owner, index): - self.session_id = session_id - self.path = path - self.value = value - self.owner = owner - self.indx = index - - -class Values(Cache): - - def getsession(self): - return util.Session() - - # value - def setvalue(self, path, value, owner, index, session): - """set value for a path - a specified value must be associated to an owner - """ - #if it's a multi - if isinstance(value, list): - value = list(value) - val = session.query(Value).filter_by( - path=path, indx=index, session_id=self._storage.session_id).first() - if val is None: - session.add(Value(self._storage.session_id, path, value, - owner, index)) - else: - val.value = value - val.owner = owner - session.commit() - - def getvalue(self, path, session, index=None): - """get value for a path - return: only value, not the owner - """ - val = session.query(Value).filter_by( - path=path, indx=index, session_id=self._storage.session_id).first() - if not val: - raise KeyError('no value found') - return val.value - - def hasvalue(self, path, session): - """if path has a value - return: boolean - """ - return session.query(Value).filter_by( - path=path, session_id=self._storage.session_id).first() is not None - - def resetvalue(self, path, session): - """remove value means delete value in storage - """ - vals = session.query(Value).filter_by( - path=path, session_id=self._storage.session_id).all() - if vals != []: - for val in vals: - session.delete(val) - session.commit() - - # owner - def setowner(self, path, owner, session, index=None): - """change owner for a path - """ - val = session.query(Value).filter_by( - path=path, indx=index, session_id=self._storage.session_id).first() - if val is None: - raise KeyError('no value found') - else: - val.owner = owner - session.commit() - - def get_max_length(self, path, session): - val = session.query(Value, func.max(Value.indx)).filter_by( - path=path, session_id=self._storage.session_id).first() - if val[1] is None: - maxval = 0 - else: - maxval = val[1] + 1 - return maxval - - def getowner(self, path, default, session, index=None, only_default=False): - #FIXME support de only_default - """get owner for a path - return: owner object - """ - session.commit() - val = session.query(Value).filter_by( - path=path, session_id=self._storage.session_id, - indx=index).first() - if val is None: - return default - else: - owner = val.owner - # autocreate owners - try: - return getattr(owners, owner) - except AttributeError: - owners.addowner(owner) - return getattr(owners, owner) - - def set_information(self, key, value): - """updates the information's attribute - (which is a dictionary) - - :param key: information's key (ex: "help", "doc" - :param value: information's value (ex: "the help string") - """ - session = self.getsession() - val = session.query(_Vinformation).filter_by( - key=key, session_id=self._storage.session_id).first() - if val is None: - session.add(_Vinformation(self._storage.session_id, key, - value)) - else: - val.value = value - session.commit() - del(session) - - def get_information(self, key, default): - """retrieves one information's item - - :param key: the item string (ex: "help") - """ - session = self.getsession() - val = session.query(_Vinformation).filter_by( - key=key, session_id=self._storage.session_id).first() - del(session) - if not val: - if default is not undefined: - return default - raise ValueError("not found") - return val.value - - def exportation(self, session, fake=False): - if fake: - #(('path1',), (index1,), (value1,), ('owner1')) - paths = [] - indexes = [] - values = [] - owners_ = [] - slaves = {} - for val in session.query(Value).filter_by( - session_id=self._storage.session_id).all(): - if val.indx is not None: - slaves.setdefault(val.path, []).append((val.indx, val.value, getattr(owners, val.owner))) - else: - paths.append(val.path) - indexes.append(val.indx) - values.append(val.value) - owners_.append(getattr(owners, val.owner)) - for path, vals in slaves.items(): - paths.append(path) - t_idxes = [] - t_vals = [] - t_owners = [] - for val in vals: - t_idxes.append(val[0]) - t_vals.append(val[1]) - t_owners.append(val[2]) - indexes.append(tuple(t_idxes)) - values.append(t_vals) - owners_.append(t_owners) - return (paths, indexes, values, owners_) - pass - - def importation(self, value): - pass - - -def delete_session(session_id, session): - for val in session.query(_Vinformation).filter_by(session_id=session_id).all(): - session.delete(val) - for val in session.query(Value).filter_by(session_id=session_id).all(): - session.delete(val) diff --git a/tiramisu/storage/sqlite3/__init__.py b/tiramisu/storage/sqlite3/__init__.py index e6034a7..2b02520 100644 --- a/tiramisu/storage/sqlite3/__init__.py +++ b/tiramisu/storage/sqlite3/__init__.py @@ -22,6 +22,12 @@ You should not configure differents Configs with same session_id. """ from .value import Values from .setting import Properties, Permissives -from .storage import Storage, list_sessions, delete_session +from .storage import SETTING, Storage, list_sessions -__all__ = ('Values', 'Properties', 'Permissives', 'Storage', 'list_sessions', 'delete_session') + +__all__ = ('SETTING', + 'Values', + 'Properties', + 'Permissives', + 'Storage', + 'list_sessions') diff --git a/tiramisu/storage/sqlite3/storage.py b/tiramisu/storage/sqlite3/storage.py index 0e0f471..09eb2b4 100644 --- a/tiramisu/storage/sqlite3/storage.py +++ b/tiramisu/storage/sqlite3/storage.py @@ -17,20 +17,23 @@ # ____________________________________________________________ from ...i18n import _ -from os import unlink -from os.path import basename, splitext, join, isfile +from os.path import join import sqlite3 -from glob import glob from ...error import ConflictError -class Setting(object): +class Setting: """:param extension: database file extension (by default: db) :param dir_database: root database directory (by default: /tmp) """ - extension = 'db' - dir_database = '/tmp' - name = 'tiramisu' + __slots__ = ('extension', + 'dir_database', + 'name') + + def __init__(self): + self.extension = 'db' + self.dir_database = '/tmp' + self.name = 'tiramisu' SETTING = Setting()