From 4336cf84a52227fcef1bc3a776e7233816612a92 Mon Sep 17 00:00:00 2001 From: Tomas Machalek Date: Tue, 18 Jun 2024 17:20:24 +0200 Subject: [PATCH] Fix and rewrite sql integration and adhoc databases Also a quic test of both integration db enabled and disabled --- conf/config.rng | 2 +- lib/action/model/user.py | 5 +- lib/action/plugin/initializer.py | 8 +- lib/plugin_types/corparch/backend/__init__.py | 3 + lib/plugin_types/integration_db.py | 122 ++----------- lib/plugins/common/mysql.rng | 3 - lib/plugins/common/mysql/__init__.py | 57 ++---- lib/plugins/common/mysql/adhocdb.py | 68 ++++---- lib/plugins/common/sqldb.py | 163 ++++++++++++++++++ .../default_integration_db/__init__.py | 30 +++- lib/plugins/mysql_auth/__init__.py | 10 +- lib/plugins/mysql_auth/config.rng | 3 - .../mysql_auth/scripts/import_users.py | 3 +- lib/plugins/mysql_auth/sign_up.py | 2 +- lib/plugins/mysql_corparch/__init__.py | 5 +- lib/plugins/mysql_corparch/backend.py | 7 +- lib/plugins/mysql_corparch/backendw.py | 2 +- lib/plugins/mysql_integration_db/__init__.py | 45 +++-- lib/plugins/mysql_integration_db/config.rng | 3 - lib/plugins/mysql_query_history/__init__.py | 18 +- .../mysql_query_persistence/__init__.py | 8 +- .../mysql_query_persistence/archive.py | 4 +- .../mysql_settings_storage/__init__.py | 14 +- lib/plugins/mysql_subc_storage/__init__.py | 19 +- lib/plugins/mysql_user_items/__init__.py | 5 +- lib/plugins/mysql_user_items/backend.py | 7 +- lib/plugins/ucnk_corparch6/config.rng | 4 - lib/plugins/ucnk_remote_auth6/config.rng | 4 - .../install/conf/docker/config.cypress.xml | 1 - scripts/install/conf/docker/config.mysql.xml | 1 - 30 files changed, 379 insertions(+), 247 deletions(-) create mode 100644 lib/plugins/common/sqldb.py diff --git a/conf/config.rng b/conf/config.rng index d9be38fcbd..1243abc2ff 100644 --- a/conf/config.rng +++ b/conf/config.rng @@ -708,7 +708,7 @@ A plugin providing connection to an existing database KonText needs to be integrated with. Typically - corparch, query_persistence, auth plug-ins may support - integration db which allows configuring and utilizing a single connection (pool) instead + integration db which allows configuring and utilizing a single connection instead of using multiple similar connections and configurations. diff --git a/lib/action/model/user.py b/lib/action/model/user.py index 2637dab571..0d2447246f 100644 --- a/lib/action/model/user.py +++ b/lib/action/model/user.py @@ -100,12 +100,11 @@ async def pre_dispatch(self, req_args): # only general setting can be applied now because # we do not know final corpus name yet self._init_default_settings(options) - + self._setup_user_paths() + self.cf = corplib.CorpusFactory(self.subcpath) try: options.update(await self._load_general_settings()) self.args.map_args_to_attrs(options) - self._setup_user_paths() - self.cf = corplib.CorpusFactory(self.subcpath) except ValueError as ex: raise UserReadableException(ex) return req_args diff --git a/lib/action/plugin/initializer.py b/lib/action/plugin/initializer.py index f2d16d6dc8..b18b5873bd 100644 --- a/lib/action/plugin/initializer.py +++ b/lib/action/plugin/initializer.py @@ -66,10 +66,14 @@ def init_plugin(name, module=None, optional=False): except ImportError as e: logging.getLogger(__name__).warning( f'Plugin [{name}] configured but the following error occurred: {e}') - except (PluginException, Exception) as e: + except PluginException as e: logging.getLogger(__name__).critical( 'Failed to initiate plug-in %s: %s', name, e, exc_info=e) - raise e + raise e from e + except Exception as e: + logging.getLogger(__name__).critical( + 'Failed to initiate plug-in %s: %s', name, e, exc_info=e) + raise PluginException(f'Failed to initiate plug-in {name} with error {e.__class__.__name__}: {e}') from e else: plugins.add_missing_plugin(name) diff --git a/lib/plugin_types/corparch/backend/__init__.py b/lib/plugin_types/corparch/backend/__init__.py index c5d0605e3f..9febb24a54 100644 --- a/lib/plugin_types/corparch/backend/__init__.py +++ b/lib/plugin_types/corparch/backend/__init__.py @@ -161,6 +161,9 @@ async def load_interval_attrs(self, cursor: CursorType, corpus_id: str) -> List[ async def load_simple_query_default_attrs(self, cursor: CursorType, corpus_id: str) -> List[str]: raise NotImplementedError() + async def close(self): + pass + class DatabaseWriteBackend(Generic[CursorType], abc.ABC): diff --git a/lib/plugin_types/integration_db.py b/lib/plugin_types/integration_db.py index 4a50dfc63c..cf3d1268cb 100644 --- a/lib/plugin_types/integration_db.py +++ b/lib/plugin_types/integration_db.py @@ -17,102 +17,14 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. import abc -from contextlib import AbstractAsyncContextManager, AbstractContextManager -from typing import Generic, Optional, TypeVar - -N = TypeVar('N') -R = TypeVar('R') -SN = TypeVar('SN') -SR = TypeVar('SR') - -T = TypeVar('T') - - -class AsyncDbContextManager(AbstractAsyncContextManager, Generic[T]): - async def __aenter__(self) -> T: - pass - - -class DbContextManager(AbstractContextManager, Generic[T]): - def __enter__(self) -> T: - pass - - -class DatabaseAdapter(abc.ABC, Generic[N, R, SN, SR]): - - @property - @abc.abstractmethod - def is_autocommit(self): - """ - Return True if autocommit is enabled for the connection; else False - """ - pass - - @property - @abc.abstractmethod - def info(self) -> str: - """ - Provide a brief info about the database. This is mainly for administrators - as it is typically written to the application log during KonText start. - """ - pass - - - @abc.abstractmethod - async def connection(self) -> AsyncDbContextManager[N]: - """ - Return an async connection to the integration database from pool. - Please note that it is important for this function not to return - a new connection each time it is called. Otherwise, functions - like commit_tx, rollback_tx won't likely work as expected. - """ - pass - - @abc.abstractmethod - async def cursor(self, dictionary=True) -> AsyncDbContextManager[R]: - """ - Create a new async database cursor - """ - pass - - @abc.abstractmethod - async def begin_tx(self, cursor): - pass - - @abc.abstractmethod - async def commit_tx(self): - """ - Commit a transaction running within - the current database connection. - """ - pass - - @abc.abstractmethod - async def rollback_tx(self): - """ - Rollback a transaction running within - the current database connection. - """ - pass - - @abc.abstractmethod - def begin_tx_sync(self, cursor): - pass - - @abc.abstractmethod - def connection_sync(self) -> DbContextManager[SN]: - pass - - @abc.abstractmethod - def cursor_sync(self, dictionary=True) -> DbContextManager[SR]: - pass +from plugins.common.sqldb import DatabaseAdapter, N, R, SN, SR +from typing import Optional class IntegrationDatabase(DatabaseAdapter[N, R, SN, SR]): """ - Integration DB plugin allows sharing a single database connection (or conn. pool) between - multiple plugins which can be convenient especially in case KonText is integrated into an existing - information system with existing user accounts, corpora information, settings storage, etc. + Integration DB plugin allows sharing a single database connection within + a scope of an HTTP action handler (i.e. one connection per HTTP request). Although not explicitly stated, the interface is tailored for use with SQL databases. where terms like 'cursor', 'commit', 'rollback' are common. But in general it should be possible @@ -121,10 +33,6 @@ class IntegrationDatabase(DatabaseAdapter[N, R, SN, SR]): Please also note that while the primary function of the plug-in is to allow integration of KonText with existing SQL databases, it can also be used to create standalone KonText installations based on MySQL/MariaDB. - - It is expected that the plugin will is able to handle a per-request connection lifecycle. - KonText helps with this by calling on_request and on_response handlers to create and close - the connection. """ @property @@ -142,34 +50,36 @@ def is_active(self): @abc.abstractmethod def wait_for_environment(self) -> Optional[Exception]: """ - This function is called each time KonText service is started + This function is called each time KonText service is started, and it should block the execution until either an internally defined/configured timeout has elapsed or some external condition allowing plug-in initialization has been fulfilled. - This can be e.g. used when KonText run within a Docker container and + This can be e.g. used when KonText run within a Docker container, and it has to wait for a database container to initialize. In such case it should e.g. try to select from a table to make sure there is a working connection along with database, tables and data. returns: - if None then the environment is ready else provide an error for further processing + if None then the environment is ready, else provide an error for further processing """ pass async def on_request(self): """ - The function is called by a Sanic 'request' middleware. - This is e.g. the right time to open a db connection - (but be aware of the scope of the connection - - see e.g. contextvars.ContextVar). + The function is called by the Sanic 'request' middleware. + This is the right place to open a db connection. + But please be aware of the scope of the connection. The plug-in + instance's scope is always "per-web worker", which means it is + shared among multiple action handlers running simultaneously. + So it is essential that the connection is scoped/isolated properly. + See e.g. Python's contextvars.ContextVar for a convenient solution. """ pass async def on_response(self): """ - The function is called by a Sanic 'response' middleware. - This is typically the right time to close the database - connection. + The function is called by the Sanic 'response' middleware. + This is the right place to close the connection. """ pass diff --git a/lib/plugins/common/mysql.rng b/lib/plugins/common/mysql.rng index a93613ca07..fb44b7e60b 100644 --- a/lib/plugins/common/mysql.rng +++ b/lib/plugins/common/mysql.rng @@ -18,9 +18,6 @@ - - - diff --git a/lib/plugins/common/mysql/__init__.py b/lib/plugins/common/mysql/__init__.py index a7cba77252..086f2ff1a6 100644 --- a/lib/plugins/common/mysql/__init__.py +++ b/lib/plugins/common/mysql/__init__.py @@ -26,31 +26,13 @@ from mysql.connector.aio.abstracts import MySQLConnectionAbstract, MySQLCursorAbstract -@dataclass -class ConnectionArgs: - db: str - user: str - password: str - autocommit: bool = False - host: str = field(default='localhost') - port: int = field(default=3306) - - -@dataclass -class PoolArgs: - minsize: int = field(default=1) - maxsize: int = field(default=10) - pool_recycle: float = field(default=-1.0) - - @dataclass class MySQLConf: database: str user: str password: str - pool_size: int - conn_retry_delay: int - conn_retry_attempts: int + retry_delay: int + retry_attempts: int host: str = field(default='localhost') port: int = field(default=3306) @@ -58,45 +40,43 @@ class MySQLConf: @staticmethod def from_conf(conf: Dict[str, Any]) -> 'MySQLConf': + pref = 'mysql_' if any(x.startswith('mysql_') for x in conf.keys()) else '' return MySQLConf( - host=conf['mysql_host'], - database=conf['mysql_db'], - user=conf['mysql_user'], - password=conf['mysql_passwd'], - pool_size=conf['mysql_pool_size'], - conn_retry_delay=conf['mysql_retry_delay'], - conn_retry_attempts=conf['mysql_retry_attempts'], + host=conf[f'{pref}host'], + database=conf[f'{pref}db'], + user=conf[f'{pref}user'], + password=conf[f'{pref}passwd'], + retry_delay=conf[f'{pref}retry_delay'], + retry_attempts=conf[f'{pref}retry_attempts'], ) @property def conn_dict(self) -> Dict[str, Any]: return asdict(self) + @property + def db(self): + return self.database + class MySQLOps: """ A simple wrapper for mysql.connector.aio and mysql.connector (for sync variants). - In KonText, the class is mostly used by the MySQLIntegrationDb plugin. + In KonText, the class is mostly used by the MySQLIntegrationDb plugin and mysql.AdhocDB. As a standalone type, it is mostly useful for running various scripts and automatic tasks on the worker server where we don't have to worry about effective connection usage (for example, if a task runs once every 5 minutes, this means - no significant connection overhead). + no significant connection overhead to always connect and disconnect). """ - _conn_args: ConnectionArgs - - _pool_args: PoolArgs + _conn_args: MySQLConf _retry_delay: int _retry_attempts: int - def __init__(self, host, database, user, password, pool_size, autocommit, retry_delay, retry_attempts): - self._conn_args = ConnectionArgs( - host=host, db=database, user=user, password=password, autocommit=autocommit) - self._pool_args = PoolArgs(maxsize=pool_size) - self._retry_delay = retry_delay # TODO has no effect now - self._retry_attempts = retry_attempts # TODO has no effect now + def __init__(self, conn_args): + self._conn_args = conn_args @asynccontextmanager async def connection(self) -> Generator[MySQLConnectionAbstract, None, None]: @@ -111,6 +91,7 @@ async def connection(self) -> Generator[MySQLConnectionAbstract, None, None]: user=self._conn_args.user, password=self._conn_args.password, host=self._conn_args.host, + port=self._conn_args.port, database=self._conn_args.db, ssl_disabled=True, autocommit=True) as conn: diff --git a/lib/plugins/common/mysql/adhocdb.py b/lib/plugins/common/mysql/adhocdb.py index c03ff548ca..667f274de9 100644 --- a/lib/plugins/common/mysql/adhocdb.py +++ b/lib/plugins/common/mysql/adhocdb.py @@ -21,24 +21,28 @@ from typing import Generator, Optional from mysql.connector.aio import connect from mysql.connector.aio.abstracts import MySQLConnectionAbstract +import uuid -from plugin_types.integration_db import AsyncDbContextManager, R, DbContextManager, SN, SR, DatabaseAdapter -from plugins.common.mysql import MySQLOps +from plugins.common.sqldb import AsyncDbContextManager, R, DbContextManager, SN, SR, DatabaseAdapter +from plugins.common.mysql import MySQLOps, MySQLConf class AdhocDB(DatabaseAdapter): + """ + AdhocDB provides a database connection for plug-ins and scripts in case IntegrationDB + is not a way to go (e.g. in systems relying on default and sqlite plugins with a single + plugin using MySQL backend). - def __init__( - self, host, database, user, password, pool_size, autocommit, retry_delay, - retry_attempts, environment_wait_sec: int - ): - self._ops = MySQLOps( - host, database, user, password, pool_size, autocommit, retry_delay, - retry_attempts - ) - self._environment_wait_sec = environment_wait_sec - self._db_conn: ContextVar[Optional[MySQLConnectionAbstract]] = ContextVar('database_connection', default=None) + !!! Important note: due to the nature of the async environment, it is super-essential + that the close() method is called by a respective plug-in's on_response event handler. + That is the only way how to ensure that each opened connection is also closed. + """ + + def __init__(self, conn_args: MySQLConf): + self._ops = MySQLOps(conn_args) + self._db_conn: ContextVar[Optional[MySQLConnectionAbstract]] = ContextVar( + f'adhocdb_{uuid.uuid1().hex}', default=None) @property def is_active(self): @@ -52,30 +56,21 @@ def is_autocommit(self): def info(self): return f'{self._ops.conn_args.host}:{self._ops.conn_args.port}/{self._ops.conn_args.db} (adhoc)' - async def on_request(self): - curr = self._db_conn.get() - if not curr: - self._db_conn.set(await self.create_connection()) - - async def on_response(self): - curr = self._db_conn.get() - if curr: - await curr.close() - async def create_connection(self) -> MySQLConnectionAbstract: return await connect( user=self._ops.conn_args.user, password=self._ops.conn_args.password, host=self._ops.conn_args.host, - database=self._ops.conn_args.db, ssl_disabled=True, autocommit=True) + database=self._ops.conn_args.database, ssl_disabled=True, autocommit=True) @asynccontextmanager async def connection(self) -> Generator[MySQLConnectionAbstract, None, None]: - curr = self._db_conn.get() - if not curr: - raise RuntimeError('No database connection') + conn = self._db_conn.get() + if not conn: + conn = await self.create_connection() + self._db_conn.set(conn) try: - yield curr + yield conn finally: - pass # No need to close the connection as it is done by Sanic middleware + pass # No need to close the connection as it is done by close() @contextmanager def connection_sync(self) -> DbContextManager[SN]: @@ -99,9 +94,6 @@ def cursor_sync(self, dictionary=True) -> DbContextManager[SR]: async def begin_tx(self, cursor): await self._ops.begin_tx(cursor) - def begin_tx_sync(self, cursor): - self._ops.begin_tx_sync(cursor) - async def commit_tx(self): async with self.connection() as conn: await conn.commit() @@ -109,3 +101,17 @@ async def commit_tx(self): async def rollback_tx(self): async with self.connection() as conn: await conn.rollback() + + def begin_tx_sync(self, cursor): + self._ops.begin_tx_sync(cursor) + + def commit_tx_sync(self, conn): + conn.commit() + + def rollback_tx_sync(self, conn): + conn.rollback() + + async def close(self): + conn = self._db_conn.get() + if conn: + await conn.close() diff --git a/lib/plugins/common/sqldb.py b/lib/plugins/common/sqldb.py new file mode 100644 index 0000000000..c501453606 --- /dev/null +++ b/lib/plugins/common/sqldb.py @@ -0,0 +1,163 @@ +# Copyright (c) 2024 Charles University, Faculty of Arts, +# Institute of the Czech National Corpus +# Copyright (c) 2024 Tomas Machalek +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# dated June, 1991. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +import abc +from abc import ABC +from contextlib import AbstractAsyncContextManager, AbstractContextManager +from typing import Generic, TypeVar + + +N = TypeVar('N') +R = TypeVar('R') +SN = TypeVar('SN') +SR = TypeVar('SR') + +T = TypeVar('T') + + +class AsyncDbContextManager(AbstractAsyncContextManager, Generic[T], ABC): + async def __aenter__(self) -> T: + pass + + +class DbContextManager(AbstractContextManager, Generic[T], ABC): + def __enter__(self) -> T: + pass + + +class DatabaseAdapter(abc.ABC, Generic[N, R, SN, SR]): + """ + DatabaseAdapter is a general purpose adapter for accessing an SQL database. + While KonText mainly relies on async database access, some parts + (miscellaneous scripts, worker jobs) rely on traditional synchronous access. + For this reason, the DatabaseAdapter interface requires both. + + Please note the important distinction between sync and async function: + + 1) the async variants expect the adapter to handle a connection internally + and keep it properly scoped and shared among multiple calls of connection(), + cursor() etc. I.e. the respective context managers are expected to lie a bit + as the connection does not exist only within the scope of a respective `with` + block. We use it rather as a reminder saying + "You are working with a connection which is handled for you". + + 2) The sync variants (e.g. connection_sync()) should create a new connection + for a respective `with` block and close the connection once the block is done + (i.e. that it the typical way how `with` is expected to behave in general). + For that matter, the sync variants of the corresponding methods take + the connection as a parameter. + """ + + @property + @abc.abstractmethod + def is_autocommit(self): + """ + Return True if autocommit is enabled for the connection; else False. + + It is expected that even with autocommit ON, it is possible to open + a transaction using the begin_tx method. + """ + pass + + @property + @abc.abstractmethod + def info(self) -> str: + """ + Provide a brief info about the database. This is mainly for administrators + as it is typically written to the application log during KonText start. + """ + pass + + + @abc.abstractmethod + async def connection(self) -> AsyncDbContextManager[N]: + """ + Return a database connection for the current scope. + Please note that it is important for this function not to return + a new connection each time it is called. Otherwise, functions + like commit_tx, rollback_tx won't likely work as expected. + I.e. it should work more like a lazy "singleton" connection provider. + """ + pass + + @abc.abstractmethod + def connection_sync(self) -> DbContextManager[SN]: + """ + This is a synchronous variant of the connection() method. + """ + pass + + @abc.abstractmethod + async def cursor(self, dictionary=True) -> AsyncDbContextManager[R]: + """ + Create a new async database cursor with scope limited + to the respective `with` block. + """ + pass + + @abc.abstractmethod + def cursor_sync(self, dictionary=True) -> DbContextManager[SR]: + """ + This is a synchronous variant of the cursor() method. + """ + pass + + @abc.abstractmethod + async def begin_tx(self, cursor: R): + """ + Start a transaction within the current connection. + """ + pass + + @abc.abstractmethod + async def commit_tx(self): + """ + Commit a transaction running within + the current database connection. + """ + pass + + @abc.abstractmethod + async def rollback_tx(self): + """ + Rollback a transaction running within + the current database connection. + """ + pass + + @abc.abstractmethod + def begin_tx_sync(self, cursor: SR): + """ + This is a synchronous variant of begin_tx + """ + pass + + @abc.abstractmethod + def commit_tx_sync(self, conn: SN): + """ + This is a synchronous variant of commit_tx + """ + pass + + @abc.abstractmethod + def rollback_tx_sync(self, conn: SN): + """ + This is a synchronous variant of rollback_tx + """ + pass + diff --git a/lib/plugins/default_integration_db/__init__.py b/lib/plugins/default_integration_db/__init__.py index 600f8e9380..c1f3324ba2 100644 --- a/lib/plugins/default_integration_db/__init__.py +++ b/lib/plugins/default_integration_db/__init__.py @@ -20,6 +20,7 @@ from contextlib import asynccontextmanager, contextmanager from plugin_types.integration_db import IntegrationDatabase +from plugins.common.sqldb import SN, SR, R from plugins.errors import PluginCompatibilityException @@ -41,26 +42,49 @@ def is_autocommit(self): @staticmethod def _err_msg(): - logging.getLogger(__name__).warning('It looks like there is no concrete integration_db enabled ' - 'and one of the active plug-ins either requires it or assumes ' - 'incorrectly that a concrete instance is enabled.') + logging.getLogger(__name__).warning( + 'It looks like there is no concrete integration_db enabled ' + 'and one of the active plug-ins either requires it or assumes ' + 'incorrectly that a concrete instance is enabled.') return 'DefaultIntegrationDb provides no true database integration' @asynccontextmanager async def connection(self): raise PluginCompatibilityException(self._err_msg()) + yield @asynccontextmanager async def cursor(self, dictionary=True): raise PluginCompatibilityException(self._err_msg()) + yield @contextmanager def connection_sync(self): raise PluginCompatibilityException(self._err_msg()) + yield @contextmanager def cursor_sync(self, dictionary=True): raise PluginCompatibilityException(self._err_msg()) + yield + + async def begin_tx(self, cursor: R): + raise PluginCompatibilityException(self._err_msg()) + + async def commit_tx(self): + raise PluginCompatibilityException(self._err_msg()) + + async def rollback_tx(self): + raise PluginCompatibilityException(self._err_msg()) + + def begin_tx_sync(self, cursor: SR): + raise PluginCompatibilityException(self._err_msg()) + + def commit_tx_sync(self, conn: SN): + raise PluginCompatibilityException(self._err_msg()) + + def rollback_tx_sync(self, conn: SN): + raise PluginCompatibilityException(self._err_msg()) @property def is_active(self): diff --git a/lib/plugins/mysql_auth/__init__.py b/lib/plugins/mysql_auth/__init__.py index c862bf1f27..1bfa930439 100644 --- a/lib/plugins/mysql_auth/__init__.py +++ b/lib/plugins/mysql_auth/__init__.py @@ -40,7 +40,7 @@ mk_pwd_hash, mk_pwd_hash_default, split_pwd_hash) from plugins import inject from plugins.common.mysql import MySQLConf -from plugin_types.integration_db import DatabaseAdapter +from plugins.common.sqldb import DatabaseAdapter from plugins.common.mysql.adhocdb import AdhocDB from plugins.mysql_corparch.backend import Backend from plugins.mysql_integration_db import MySqlIntegrationDb @@ -343,6 +343,10 @@ async def get_form_props_from_token(self, key): return token.user return None + async def on_response(self): + if isinstance(self.db, AdhocDB): + await self.db.close() + @inject(plugins.runtime.INTEGRATION_DB) def create_instance(conf, integ_db: MySqlIntegrationDb): @@ -356,11 +360,11 @@ def create_instance(conf, integ_db: MySqlIntegrationDb): logging.getLogger(__name__).info(f'mysql_auth uses integration_db[{integ_db.info}]') corparch_backend = Backend(integ_db, enable_parallel_acc=True) else: - dbx = AdhocDB(**MySQLConf.from_conf(plugin_conf).conn_dict) + dbx = AdhocDB(MySQLConf.from_conf(plugin_conf)) logging.getLogger(__name__).info( 'mysql_auth uses custom database configuration {}@{}'.format( plugin_conf['mysql_user'], plugin_conf['mysql_host'])) - corparch_backend = Backend(AdhocDB(**MySQLConf.from_conf(plugin_conf).conn_dict)) + corparch_backend = Backend(dbx) return MysqlAuthHandler( db=dbx, corparch_backend=corparch_backend, diff --git a/lib/plugins/mysql_auth/config.rng b/lib/plugins/mysql_auth/config.rng index 4579e8afaf..a5914a08eb 100644 --- a/lib/plugins/mysql_auth/config.rng +++ b/lib/plugins/mysql_auth/config.rng @@ -63,9 +63,6 @@ - - - diff --git a/lib/plugins/mysql_auth/scripts/import_users.py b/lib/plugins/mysql_auth/scripts/import_users.py index 459e1fe597..bcf03ba1ee 100644 --- a/lib/plugins/mysql_auth/scripts/import_users.py +++ b/lib/plugins/mysql_auth/scripts/import_users.py @@ -46,11 +46,12 @@ def import_user(data): cursor.execute( 'INSERT INTO kontext_user_access (user_id, corpus_name, limited) ' 'VALUES (%s, %s, 0)', (user_id, corp)) - conn.commit() + auth.db.commit_tx_sync(conn) print(('Installed user {}'.format(data['username']))) return 1 except Exception as ex: print(ex) + auth.db.rollback_tx_sync(conn) return 0 diff --git a/lib/plugins/mysql_auth/sign_up.py b/lib/plugins/mysql_auth/sign_up.py index 6817e3412f..99bea15e84 100644 --- a/lib/plugins/mysql_auth/sign_up.py +++ b/lib/plugins/mysql_auth/sign_up.py @@ -18,7 +18,7 @@ from mysql.connector.aio.abstracts import MySQLConnectionAbstract from plugin_types.auth.sign_up import AbstractSignUpToken -from plugin_types.integration_db import DatabaseAdapter +from plugins.common.sqldb import DatabaseAdapter class SignUpToken(AbstractSignUpToken[MySQLConnectionAbstract]): diff --git a/lib/plugins/mysql_corparch/__init__.py b/lib/plugins/mysql_corparch/__init__.py index 0e61724c48..ca2c1d1484 100644 --- a/lib/plugins/mysql_corparch/__init__.py +++ b/lib/plugins/mysql_corparch/__init__.py @@ -424,6 +424,9 @@ async def export(self, plugin_ctx): max_page_size=self.max_page_size ) + async def on_response(self): + await self.backend.close() + @inject(plugins.runtime.USER_ITEMS, plugins.runtime.INTEGRATION_DB) def create_instance(conf, user_items: AbstractUserItems, integ_db: MySqlIntegrationDb): @@ -435,7 +438,7 @@ def create_instance(conf, user_items: AbstractUserItems, integ_db: MySqlIntegrat logging.getLogger(__name__).info( 'mysql_user_items uses custom database configuration {}@{}'.format( plugin_conf['mysql_user'], plugin_conf['mysql_host'])) - db_backend = Backend(AdhocDB(**MySQLConf.from_conf(plugin_conf).conn_dict)) + db_backend = Backend(AdhocDB(MySQLConf.from_conf(plugin_conf))) return MySQLCorparch( db_backend=db_backend, diff --git a/lib/plugins/mysql_corparch/backend.py b/lib/plugins/mysql_corparch/backend.py index 36786c854a..e3c67eb0c8 100644 --- a/lib/plugins/mysql_corparch/backend.py +++ b/lib/plugins/mysql_corparch/backend.py @@ -31,7 +31,8 @@ POS_COLS_MAP, REG_COLS_MAP, REG_VAR_COLS_MAP, SATTR_COLS_MAP, STRUCT_COLS_MAP) from plugin_types.corparch.corpus import PosCategoryItem, TagsetInfo -from plugin_types.integration_db import DatabaseAdapter +from plugins.common.sqldb import DatabaseAdapter +from plugins.common.mysql.adhocdb import AdhocDB class MySQLConfException(Exception): @@ -511,3 +512,7 @@ async def load_simple_query_default_attrs(self, cursor: MySQLCursorAbstract, cor 'SELECT pos_attr FROM kontext_simple_query_default_attrs WHERE corpus_name = %s', (corpus_id,)) return [r['pos_attr'] for r in await cursor.fetchall()] + + async def close(self): + if isinstance(self._db, AdhocDB): + await self._db.close() diff --git a/lib/plugins/mysql_corparch/backendw.py b/lib/plugins/mysql_corparch/backendw.py index 4cffeb79d2..47557e1b4e 100644 --- a/lib/plugins/mysql_corparch/backendw.py +++ b/lib/plugins/mysql_corparch/backendw.py @@ -31,7 +31,7 @@ REG_VAR_COLS_MAP, SATTR_COLS_MAP, STRUCT_COLS_MAP) -from plugin_types.integration_db import DatabaseAdapter +from plugins.common.sqldb import DatabaseAdapter from plugins.mysql_corparch.backend import ( DFLT_CORP_TABLE, DFLT_GROUP_ACC_CORP_ATTR, diff --git a/lib/plugins/mysql_integration_db/__init__.py b/lib/plugins/mysql_integration_db/__init__.py index 6d231f6676..8c1eccc830 100644 --- a/lib/plugins/mysql_integration_db/__init__.py +++ b/lib/plugins/mysql_integration_db/__init__.py @@ -25,12 +25,19 @@ from mysql.connector.aio.abstracts import MySQLConnectionAbstract, MySQLCursorAbstract from mysql.connector.abstracts import MySQLConnectionAbstract as SyncMySQLConnectionAbstract, MySQLCursorAbstract as SyncMySQLCursorAbstract -from plugin_types.integration_db import IntegrationDatabase, AsyncDbContextManager, R, DbContextManager, SN, SR -from plugins.common.mysql import MySQLOps - - -class MySqlIntegrationDb(IntegrationDatabase[MySQLConnectionAbstract, MySQLCursorAbstract, SyncMySQLConnectionAbstract, SyncMySQLCursorAbstract]): - """ +from plugins.common.sqldb import AsyncDbContextManager, R, DbContextManager, SN, SR +from plugins.common.mysql import MySQLOps, MySQLConf +from plugin_types.integration_db import IntegrationDatabase + + +class MySqlIntegrationDb( + IntegrationDatabase[ + MySQLConnectionAbstract, + MySQLCursorAbstract, + SyncMySQLConnectionAbstract, + SyncMySQLCursorAbstract + ]): + """s MySqlIntegrationDb is a variant of integration_db plug-in providing access to MySQL/MariaDB instances. It is recommended for: 1) integration with existing MySQL/MariaDB information systems, @@ -46,13 +53,9 @@ class MySqlIntegrationDb(IntegrationDatabase[MySQLConnectionAbstract, MySQLCurso """ def __init__( - self, host, database, user, password, pool_size, autocommit, retry_delay, - retry_attempts, environment_wait_sec: int + self, conn_args: MySQLConf, environment_wait_sec: int ): - self._ops = MySQLOps( - host, database, user, password, pool_size, autocommit, retry_delay, - retry_attempts - ) + self._ops = MySQLOps(conn_args) self._environment_wait_sec = environment_wait_sec self._db_conn: ContextVar[Optional[MySQLConnectionAbstract]] = ContextVar('database_connection', default=None) @@ -101,7 +104,7 @@ async def on_response(self): async def create_connection(self) -> MySQLConnectionAbstract: return await connect( user=self._ops.conn_args.user, password=self._ops.conn_args.password, host=self._ops.conn_args.host, - database=self._ops.conn_args.db, ssl_disabled=True, autocommit=True) + database=self._ops.conn_args.database, ssl_disabled=True, autocommit=True) @asynccontextmanager async def connection(self) -> Generator[MySQLConnectionAbstract, None, None]: @@ -135,9 +138,6 @@ def cursor_sync(self, dictionary=True) -> DbContextManager[SR]: async def begin_tx(self, cursor): await self._ops.begin_tx(cursor) - def begin_tx_sync(self, cursor): - self._ops.begin_tx_sync(cursor) - async def commit_tx(self): async with self.connection() as conn: await conn.commit() @@ -146,11 +146,18 @@ async def rollback_tx(self): async with self.connection() as conn: await conn.rollback() + def begin_tx_sync(self, cursor): + self._ops.begin_tx_sync(cursor) + + def commit_tx_sync(self, conn): + conn.commit() + + def rollback_tx_sync(self, conn): + conn.rollback() + def create_instance(conf): pconf = conf.get('plugins', 'integration_db') return MySqlIntegrationDb( - host=pconf['host'], database=pconf['db'], user=pconf['user'], password=pconf['passwd'], - pool_size=int(pconf['pool_size']), autocommit=True, - retry_delay=int(pconf['retry_delay']), retry_attempts=int(pconf['retry_attempts']), + MySQLConf.from_conf(pconf), environment_wait_sec=int(pconf['environment_wait_sec'])) diff --git a/lib/plugins/mysql_integration_db/config.rng b/lib/plugins/mysql_integration_db/config.rng index 4d9019d8e1..7e40c1e9d1 100644 --- a/lib/plugins/mysql_integration_db/config.rng +++ b/lib/plugins/mysql_integration_db/config.rng @@ -19,9 +19,6 @@ - - - diff --git a/lib/plugins/mysql_query_history/__init__.py b/lib/plugins/mysql_query_history/__init__.py index 1e77d5b9a5..b5376c9022 100644 --- a/lib/plugins/mysql_query_history/__init__.py +++ b/lib/plugins/mysql_query_history/__init__.py @@ -42,6 +42,8 @@ from plugin_types.subc_storage import AbstractSubcArchive from plugins import inject from plugins.mysql_integration_db import MySqlIntegrationDb +from plugins.common.mysql.adhocdb import AdhocDB +from plugins.common.mysql import MySQLConf class CorpusCache: @@ -359,6 +361,10 @@ def export_tasks(self): """ return self.delete_old_records, + async def on_response(self): + if isinstance(self._db, AdhocDB): + await self._db.close() + @inject( plugins.runtime.INTEGRATION_DB, @@ -367,10 +373,16 @@ def export_tasks(self): plugins.runtime.AUTH ) def create_instance( - settings, - db: MySqlIntegrationDb, + conf, + integ_db: MySqlIntegrationDb, query_persistence: AbstractQueryPersistence, subc_archive: AbstractSubcArchive, auth: AbstractAuth ): - return MySqlQueryHistory(settings, db, query_persistence, subc_archive, auth) + plugin_conf = conf.get('plugins', 'auth') + if integ_db and integ_db.is_active and 'mysql_host' not in plugin_conf: + db = integ_db + logging.getLogger(__name__).info(f'mysql_query_history uses integration_db[{integ_db.info}]') + else: + db = AdhocDB(MySQLConf.from_conf(plugin_conf)) + return MySqlQueryHistory(conf, db, query_persistence, subc_archive, auth) diff --git a/lib/plugins/mysql_query_persistence/__init__.py b/lib/plugins/mysql_query_persistence/__init__.py index 31fcc11eef..1d4e6723de 100644 --- a/lib/plugins/mysql_query_persistence/__init__.py +++ b/lib/plugins/mysql_query_persistence/__init__.py @@ -77,7 +77,7 @@ from plugins import inject from plugins.common.mysql import MySQLConf from plugins.common.mysql.adhocdb import AdhocDB -from plugin_types.integration_db import DatabaseAdapter +from plugins.common.sqldb import DatabaseAdapter from plugins.mysql_integration_db import MySqlIntegrationDb from .archive import get_iso_datetime, is_archived @@ -399,6 +399,10 @@ async def id_exists(self, id: str) -> bool: row = await cursor.fetchone() return row is not None or await self.db.exists(mk_key(id)) + async def on_response(self): + if isinstance(self._archive, AdhocDB): + await self._archive.close() + @inject(plugins.runtime.DB, plugins.runtime.INTEGRATION_DB, plugins.runtime.AUTH) def create_instance(settings, db: KeyValueStorage, integration_db: MySqlIntegrationDb, auth: AbstractAuth): @@ -414,4 +418,4 @@ def create_instance(settings, db: KeyValueStorage, integration_db: MySqlIntegrat logging.getLogger(__name__).info( 'mysql_query_persistence uses custom database configuration {}@{}'.format( plugin_conf['mysql_user'], plugin_conf['mysql_host'])) - return MySqlQueryPersistence(settings, db, AdhocDB(**MySQLConf.from_conf(plugin_conf).conn_dict), auth) + return MySqlQueryPersistence(settings, db, AdhocDB(MySQLConf.from_conf(plugin_conf)), auth) diff --git a/lib/plugins/mysql_query_persistence/archive.py b/lib/plugins/mysql_query_persistence/archive.py index 0ddd1cfc8e..0457b7d839 100644 --- a/lib/plugins/mysql_query_persistence/archive.py +++ b/lib/plugins/mysql_query_persistence/archive.py @@ -26,7 +26,7 @@ import ujson as json from mysql.connector.aio.abstracts import MySQLCursorAbstract from plugin_types.general_storage import KeyValueStorage -from plugin_types.integration_db import DatabaseAdapter +from plugins.common.sqldb import DatabaseAdapter def get_iso_datetime(): @@ -41,7 +41,7 @@ async def is_archived(cursor: MySQLCursorAbstract, conc_id): return (await cursor.fetchone()) is not None -class Archiver(object): +class Archiver: """ A class which actually performs the process of archiving records from fast database (Redis) to a slow one (SQLite3) diff --git a/lib/plugins/mysql_settings_storage/__init__.py b/lib/plugins/mysql_settings_storage/__init__.py index 7b1029727b..6cc3ae4386 100644 --- a/lib/plugins/mysql_settings_storage/__init__.py +++ b/lib/plugins/mysql_settings_storage/__init__.py @@ -27,6 +27,8 @@ from plugin_types.settings_storage import AbstractSettingsStorage from plugins import inject from plugins.mysql_integration_db import MySqlIntegrationDb +from plugins.common.mysql.adhocdb import AdhocDB +from plugins.common.mysql import MySQLConf class SettingsStorage(AbstractSettingsStorage): @@ -110,13 +112,23 @@ async def load(self, user_id: int, corpus_id: Dict[str, Serializable] = None): def get_excluded_users(self): return self._excluded_users + async def on_response(self): + if isinstance(self._db, AdhocDB): + await self._db.close() + @inject(plugins.runtime.INTEGRATION_DB) -def create_instance(conf, db: MySqlIntegrationDb): +def create_instance(conf, integration_db: MySqlIntegrationDb): conf = conf.get('plugins', 'settings_storage') excluded_users = conf.get('excluded_users', None) if excluded_users is None: excluded_users = [] else: excluded_users = [int(x) for x in excluded_users] + if integration_db.is_active and 'mysql_host' not in conf: + db = integration_db + logging.getLogger(__name__).info( + f'mysql_settings_storage uses integration_db[{integration_db.info}]') + else: + db = AdhocDB(MySQLConf.from_conf(conf)) return SettingsStorage(db, excluded_users=excluded_users) diff --git a/lib/plugins/mysql_subc_storage/__init__.py b/lib/plugins/mysql_subc_storage/__init__.py index 11ee8794fe..3b0cd849fb 100644 --- a/lib/plugins/mysql_subc_storage/__init__.py +++ b/lib/plugins/mysql_subc_storage/__init__.py @@ -13,7 +13,6 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. - import logging import os import struct @@ -31,7 +30,8 @@ from plugin_types.corparch import AbstractCorporaArchive from plugin_types.subc_storage import AbstractSubcArchive, SubcArchiveException from plugins import inject -from plugins.errors import PluginCompatibilityException +from plugins.common.mysql.adhocdb import AdhocDB +from plugins.common.mysql import MySQLConf from plugins.mysql_integration_db import MySqlIntegrationDb from mysql.connector.errors import IntegrityError from sanic import Sanic @@ -392,13 +392,18 @@ async def update_name_and_description(self, user_id: int, subc_id: str, subcname ) return k_markdown(description) + async def on_response(self): + if isinstance(self._db, AdhocDB): + await self._db.close() + @inject(plugins.runtime.CORPARCH, plugins.runtime.INTEGRATION_DB) def create_instance(conf, corparch: AbstractCorporaArchive, integ_db: MySqlIntegrationDb): plugin_conf = conf.get('plugins', 'subc_storage') - if integ_db.is_active: - logging.getLogger(__name__).info(f'mysql_subc_storage uses integration_db[{integ_db.info}]') - return MySQLSubcArchive(plugin_conf, corparch, integ_db, BackendConfig()) + if integ_db.is_active and 'mysql_host' not in plugin_conf: + db = integ_db + logging.getLogger(__name__).info( + f'mysql_subc_storage uses integration_db[{integ_db.info}]') else: - raise PluginCompatibilityException( - 'mysql_subc_storage works only with integration_db enabled') + db = AdhocDB(MySQLConf.from_conf(plugin_conf)) + return MySQLSubcArchive(plugin_conf, corparch, db, BackendConfig()) diff --git a/lib/plugins/mysql_user_items/__init__.py b/lib/plugins/mysql_user_items/__init__.py index a4efeb230d..aa2704cb66 100644 --- a/lib/plugins/mysql_user_items/__init__.py +++ b/lib/plugins/mysql_user_items/__init__.py @@ -143,6 +143,9 @@ def export_actions(): def max_num_favorites(self): return int(self._settings.get('plugins', 'user_items')['max_num_favorites']) + async def on_response(self): + await self._backend.close() + @inject(plugins.runtime.INTEGRATION_DB, plugins.runtime.AUTH) def create_instance(settings, integ_db: MySqlIntegrationDb, auth: AbstractAuth): @@ -154,5 +157,5 @@ def create_instance(settings, integ_db: MySqlIntegrationDb, auth: AbstractAuth): logging.getLogger(__name__).info( 'mysql_user_items uses custom database configuration {}@{}'.format( plugin_conf['mysql_user'], plugin_conf['mysql_host'])) - db_backend = Backend(AdhocDB(**MySQLConf.from_conf(plugin_conf).conn_dict)) + db_backend = Backend(AdhocDB(MySQLConf.from_conf(plugin_conf))) return MySQLUserItems(settings, db_backend, auth) diff --git a/lib/plugins/mysql_user_items/backend.py b/lib/plugins/mysql_user_items/backend.py index 5b517956fa..8c357f3930 100644 --- a/lib/plugins/mysql_user_items/backend.py +++ b/lib/plugins/mysql_user_items/backend.py @@ -23,7 +23,8 @@ from typing import List from plugin_types.user_items import FavoriteItem -from plugin_types.integration_db import DatabaseAdapter +from plugins.common.sqldb import DatabaseAdapter +from plugins.common.mysql.adhocdb import AdhocDB DFLT_USER_TABLE = 'kontext_user' DFLT_CORP_TABLE = 'kontext_corpus' @@ -107,3 +108,7 @@ async def delete_favitem(self, item_id: int): 'DELETE FROM kontext_corpus_user_fav_item WHERE user_fav_corpus_id = %s', (item_id,)) await cursor.execute('DELETE FROM kontext_user_fav_item WHERE id = %s', (item_id,)) await conn.commit() + + async def close(self): + if isinstance(self._db, AdhocDB): + await self._db.close() diff --git a/lib/plugins/ucnk_corparch6/config.rng b/lib/plugins/ucnk_corparch6/config.rng index 6ae986f28d..4071c69ee1 100644 --- a/lib/plugins/ucnk_corparch6/config.rng +++ b/lib/plugins/ucnk_corparch6/config.rng @@ -52,10 +52,6 @@ - - 1 should be OK in most cases - - diff --git a/lib/plugins/ucnk_remote_auth6/config.rng b/lib/plugins/ucnk_remote_auth6/config.rng index b282abff73..caf1c64da8 100644 --- a/lib/plugins/ucnk_remote_auth6/config.rng +++ b/lib/plugins/ucnk_remote_auth6/config.rng @@ -71,10 +71,6 @@ - - 1 should be OK in most cases - - diff --git a/scripts/install/conf/docker/config.cypress.xml b/scripts/install/conf/docker/config.cypress.xml index a43536df95..c2bfb3ffac 100644 --- a/scripts/install/conf/docker/config.cypress.xml +++ b/scripts/install/conf/docker/config.cypress.xml @@ -123,7 +123,6 @@ kontext kontext kontext-secret - 3 2 5 10 diff --git a/scripts/install/conf/docker/config.mysql.xml b/scripts/install/conf/docker/config.mysql.xml index 281edcef40..0fab7d1eeb 100644 --- a/scripts/install/conf/docker/config.mysql.xml +++ b/scripts/install/conf/docker/config.mysql.xml @@ -122,7 +122,6 @@ kontext kontext kontext-secret - 3 2 5 10