user-sessions: added an API call to cleanup sessions.

rhodecode/api/tests/test_cleanup_sessions.py

@@ -0,0 +1,44 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2017-2017 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import mock
+import pytest
+
+from rhodecode.lib.user_sessions import FileAuthSessions
+from rhodecode.api.tests.utils import (
+    build_data, api_call, assert_ok, assert_error, crash)
+
+
[e-mail]("testuser_api", "app")
+class TestCleanupSessions(object):
+    def test_api_cleanup_sessions(self):
+        id_, params = build_data(self.apikey, 'cleanup_sessions')
+        response = api_call(self.app, params)
+
+        expected = {'backend': 'file sessions', 'sessions_removed': 0}
+        assert_ok(id_, expected, given=response.body)
+
+    @mock.patch.object(FileAuthSessions, 'clean_sessions', crash)
+    def test_api_cleanup_error(self):
+        id_, params = build_data(self.apikey, 'cleanup_sessions', )
+        response = api_call(self.app, params)
+
+        expected = 'Error occurred during session cleanup'
+        assert_error(id_, expected, given=response.body)

rhodecode/api/views/server_api.py

@@ -1,178 +1,244 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import logging
 
 from rhodecode.api import jsonrpc_method, JSONRPCError, JSONRPCForbidden
 
 from rhodecode.api.utils import (
     Optional, OAttr, has_superadmin_permission, get_user_or_error)
 from rhodecode.lib.utils import repo2db_mapper
+from rhodecode.lib import system_info
+from rhodecode.lib import user_sessions
 from rhodecode.model.db import UserIpMap
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
 
 
 @jsonrpc_method()
 def get_server_info(request, apiuser):
     """
     Returns the |RCE| server information.
 
     This includes the running version of |RCE| and all installed
     packages. This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'modules': [<module name>,...]
         'py_version': <python version>,
         'platform': <platform type>,
         'rhodecode_version': <rhodecode version>
       }
       error :  null
     """
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     server_info = ScmModel().get_server_info(request.environ)
     # rhodecode-index requires those
 
     server_info['index_storage'] = server_info['search']['value']['location']
     server_info['storage'] = server_info['storage']['value']['path']
 
     return server_info
 
 
 @jsonrpc_method()
 def get_ip(request, apiuser, userid=Optional(OAttr('apiuser'))):
     """
     Displays the IP Address as seen from the |RCE| server.
 
     * This command displays the IP Address, as well as all the defined IP
       addresses for the specified user. If the ``userid`` is not set, the
       data returned is for the user calling the method.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from |authtoken|.
     :type apiuser: AuthUser
     :param userid: Sets the userid for which associated IP Address data
         is returned.
     :type userid: Optional(str or int)
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result : {
                      "server_ip_addr": "<ip_from_clien>",
                      "user_ips": [
                                     {
                                        "ip_addr": "<ip_with_mask>",
                                        "ip_range": ["<start_ip>", "<end_ip>"],
                                     },
                                     ...
                                  ]
         }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     userid = Optional.extract(userid, evaluate_locals=locals())
     userid = getattr(userid, 'user_id', userid)
 
     user = get_user_or_error(userid)
     ips = UserIpMap.query().filter(UserIpMap.user == user).all()
     return {
         'server_ip_addr': request.rpc_ip_addr,
         'user_ips': ips
     }
 
 
 @jsonrpc_method()
 def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
     """
     Triggers a rescan of the specified repositories.
 
     * If the ``remove_obsolete`` option is set, it also deletes repositories
       that are found in the database but not on the file system, so called
       "clean zombies".
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param remove_obsolete: Deletes repositories from the database that
         are not found on the filesystem.
     :type remove_obsolete: Optional(``True`` | ``False``)
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'added': [<added repository name>,...]
         'removed': [<removed repository name>,...]
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         'Error occurred during rescan repositories action'
       }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     try:
         rm_obsolete = Optional.extract(remove_obsolete)
         added, removed = repo2db_mapper(ScmModel().repo_scan(),
                                         remove_obsolete=rm_obsolete)
         return {'added': added, 'removed': removed}
     except Exception:
         log.exception('Failed to run repo rescann')
         raise JSONRPCError(
             'Error occurred during rescan repositories action'
         )
 
+
+@jsonrpc_method()
+def cleanup_sessions(request, apiuser, older_then=Optional(60)):
+    """
+    Triggers a session cleanup action.
+
+    If the ``older_then`` option is set, only sessions that hasn't been
+    accessed in the given number of days will be removed.
+
+    This command can only be run using an |authtoken| with admin rights to
+    the specified repository.
+
+    This command takes the following options:
+
+    :param apiuser: This is filled automatically from the |authtoken|.
+    :type apiuser: AuthUser
+    :param older_then: Deletes session that hasn't been accessed
+        in given number of days.
+    :type older_then: Optional(int)
+
+    Example output:
+
+    .. code-block:: bash
+
+      id : <id_given_in_input>
+      result: {
+        "backend": "<type of backend>",
+        "sessions_removed": <number_of_removed_sessions>
+      }
+      error :  null
+
+    Example error output:
+
+    .. code-block:: bash
+
+      id : <id_given_in_input>
+      result : null
+      error :  {
+        'Error occurred during session cleanup'
+      }
+
+    """
+    if not has_superadmin_permission(apiuser):
+        raise JSONRPCForbidden()
+
+    older_then = Optional.extract(older_then)
+    older_than_seconds = 60 * 60 * 24 * older_then
+
+    config = system_info.rhodecode_config().get_value()['value']['config']
+    session_model = user_sessions.get_session_handler(
+        config.get('beaker.session.type', 'memory'))(config)
+
+    backend = session_model.SESSION_TYPE
+    try:
+        cleaned = session_model.clean_sessions(
+            older_than_seconds=older_than_seconds)
+        return {'sessions_removed': cleaned, 'backend': backend}
+    except user_sessions.CleanupCommand as msg:
+        return {'cleanup_command': msg.message, 'backend': backend}
+    except Exception as e:
+        log.exception('Failed session cleanup')
+        raise JSONRPCError(
+            'Error occurred during session cleanup'
+        )

rhodecode/lib/user_sessions.py

@@ -1,174 +1,181 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2017-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import time
 import datetime
 import dateutil
 from rhodecode.model.db import DbSession, Session
 
 
 class CleanupCommand(Exception):
     pass
 
 
 class BaseAuthSessions(object):
     SESSION_TYPE = None
     NOT_AVAILABLE = 'NOT AVAILABLE'
 
     def __init__(self, config):
         session_conf = {}
         for k, v in config.items():
             if k.startswith('beaker.session'):
                 session_conf[k] = v
         self.config = session_conf
 
     def get_count(self):
         raise NotImplementedError
 
     def get_expired_count(self, older_than_seconds=None):
         raise NotImplementedError
 
     def clean_sessions(self, older_than_seconds=None):
         raise NotImplementedError
 
     def _seconds_to_date(self, seconds):
         return datetime.datetime.utcnow() - dateutil.relativedelta.relativedelta(
             seconds=seconds)
 
 
 class DbAuthSessions(BaseAuthSessions):
     SESSION_TYPE = 'ext:database'
 
     def get_count(self):
         return DbSession.query().count()
 
     def get_expired_count(self, older_than_seconds=None):
         expiry_date = self._seconds_to_date(older_than_seconds)
         return DbSession.query().filter(DbSession.accessed < expiry_date).count()
 
     def clean_sessions(self, older_than_seconds=None):
         expiry_date = self._seconds_to_date(older_than_seconds)
+        to_remove = DbSession.query().filter(DbSession.accessed < expiry_date).count()
         DbSession.query().filter(DbSession.accessed < expiry_date).delete()
         Session().commit()
+        return to_remove
 
 
 class FileAuthSessions(BaseAuthSessions):
     SESSION_TYPE = 'file sessions'
 
     def _get_sessions_dir(self):
         data_dir = self.config.get('beaker.session.data_dir')
         return data_dir
 
     def _count_on_filesystem(self, path, older_than=0, callback=None):
-        value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
+        value = dict(percent=0, used=0, total=0, items=0, callbacks=0,
+                     path=path, text='')
         items_count = 0
         used = 0
+        callbacks = 0
         cur_time = time.time()
         for root, dirs, files in os.walk(path):
             for f in files:
                 final_path = os.path.join(root, f)
                 try:
                     mtime = os.stat(final_path).st_mtime
                     if (cur_time - mtime) > older_than:
                         items_count += 1
                         if callback:
                             callback_res = callback(final_path)
+                            callbacks += 1
                         else:
                             used += os.path.getsize(final_path)
                 except OSError:
                     pass
         value.update({
             'percent': 100,
             'used': used,
             'total': used,
-            'items': items_count
+            'items': items_count,
+            'callbacks': callbacks
         })
         return value
 
     def get_count(self):
         try:
             sessions_dir = self._get_sessions_dir()
             items_count = self._count_on_filesystem(sessions_dir)['items']
         except Exception:
             items_count = self.NOT_AVAILABLE
         return items_count
 
     def get_expired_count(self, older_than_seconds=0):
         try:
             sessions_dir = self._get_sessions_dir()
             items_count = self._count_on_filesystem(
                 sessions_dir, older_than=older_than_seconds)['items']
         except Exception:
             items_count = self.NOT_AVAILABLE
         return items_count
 
     def clean_sessions(self, older_than_seconds=0):
         # find . -mtime +60 -exec rm {} \;
 
         sessions_dir = self._get_sessions_dir()
 
         def remove_item(path):
             os.remove(path)
 
-        return self._count_on_filesystem(
+        stats = self._count_on_filesystem(
             sessions_dir, older_than=older_than_seconds,
-            callback=remove_item)['items']
+            callback=remove_item)
+        return stats['callbacks']
 
 
 class MemcachedAuthSessions(BaseAuthSessions):
     SESSION_TYPE = 'ext:memcached'
 
     def get_count(self):
         return self.NOT_AVAILABLE
 
     def get_expired_count(self, older_than_seconds=None):
         return self.NOT_AVAILABLE
 
     def clean_sessions(self, older_than_seconds=None):
         raise CleanupCommand('Cleanup for this session type not yet available')
 
 
 class MemoryAuthSessions(BaseAuthSessions):
     SESSION_TYPE = 'memory'
 
     def get_count(self):
         return self.NOT_AVAILABLE
 
     def get_expired_count(self, older_than_seconds=None):
         return self.NOT_AVAILABLE
 
     def clean_sessions(self, older_than_seconds=None):
         raise CleanupCommand('Cleanup for this session type not yet available')
 
 
 def get_session_handler(session_type):
     types = {
         'file': FileAuthSessions,
         'ext:memcached': MemcachedAuthSessions,
         'ext:database': DbAuthSessions,
         'memory': MemoryAuthSessions
     }
 
     try:
         return types[session_type]
     except KeyError:
         raise ValueError(
             'This type {} is not supported'.format(session_type))