compat: use py3 compatible configparser in all places.

rhodecode/apps/ssh_support/lib/backends/__init__.py

@@ -1,207 +1,207
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import re
 import logging
 import datetime
-import ConfigParser
+from pyramid.compat import configparser
 
 from rhodecode.model.db import Session, User, UserSshKeys
 from rhodecode.model.scm import ScmModel
 
 from .hg import MercurialServer
 from .git import GitServer
 from .svn import SubversionServer
 log = logging.getLogger(__name__)
 
 
 class SshWrapper(object):
 
     def __init__(self, command, connection_info, mode,
                  user, user_id, key_id, shell, ini_path, env):
         self.command = command
         self.connection_info = connection_info
         self.mode = mode
         self.user = user
         self.user_id = user_id
         self.key_id = key_id
         self.shell = shell
         self.ini_path = ini_path
         self.env = env
 
         self.config = self.parse_config(ini_path)
         self.server_impl = None
 
     def parse_config(self, config_path):
-        parser = ConfigParser.ConfigParser()
+        parser = configparser.ConfigParser()
         parser.read(config_path)
         return parser
 
     def update_key_access_time(self, key_id):
         key = UserSshKeys().query().filter(
             UserSshKeys.ssh_key_id == key_id).scalar()
         if key:
             key.accessed_on = datetime.datetime.utcnow()
             Session().add(key)
             Session().commit()
             log.debug('Update key `%s` access time', key_id)
 
     def get_connection_info(self):
         """
         connection_info
 
         Identifies the client and server ends of the connection.
         The variable contains four space-separated values: client IP address,
         client port number, server IP address, and server port number.
         """
         conn = dict(
             client_ip=None,
             client_port=None,
             server_ip=None,
             server_port=None,
         )
 
         info = self.connection_info.split(' ')
         if len(info) == 4:
             conn['client_ip'] = info[0]
             conn['client_port'] = info[1]
             conn['server_ip'] = info[2]
             conn['server_port'] = info[3]
 
         return conn
 
     def get_repo_details(self, mode):
         vcs_type = mode if mode in ['svn', 'hg', 'git'] else None
         mode = mode
         repo_name = None
 
         hg_pattern = r'^hg\s+\-R\s+(\S+)\s+serve\s+\-\-stdio$'
         hg_match = re.match(hg_pattern, self.command)
         if hg_match is not None:
             vcs_type = 'hg'
             repo_name = hg_match.group(1).strip('/')
             return vcs_type, repo_name, mode
 
         git_pattern = (
             r'^git-(receive-pack|upload-pack)\s\'[/]?(\S+?)(|\.git)\'$')
         git_match = re.match(git_pattern, self.command)
         if git_match is not None:
             vcs_type = 'git'
             repo_name = git_match.group(2).strip('/')
             mode = git_match.group(1)
             return vcs_type, repo_name, mode
 
         svn_pattern = r'^svnserve -t'
         svn_match = re.match(svn_pattern, self.command)
 
         if svn_match is not None:
             vcs_type = 'svn'
             # Repo name should be extracted from the input stream
             return vcs_type, repo_name, mode
 
         return vcs_type, repo_name, mode
 
     def serve(self, vcs, repo, mode, user, permissions):
         store = ScmModel().repos_path
 
         log.debug(
             'VCS detected:`%s` mode: `%s` repo_name: %s', vcs, mode, repo)
 
         if vcs == 'hg':
             server = MercurialServer(
                 store=store, ini_path=self.ini_path,
                 repo_name=repo, user=user,
                 user_permissions=permissions, config=self.config, env=self.env)
             self.server_impl = server
             return server.run()
 
         elif vcs == 'git':
             server = GitServer(
                 store=store, ini_path=self.ini_path,
                 repo_name=repo, repo_mode=mode, user=user,
                 user_permissions=permissions, config=self.config, env=self.env)
             self.server_impl = server
             return server.run()
 
         elif vcs == 'svn':
             server = SubversionServer(
                 store=store, ini_path=self.ini_path,
                 repo_name=None, user=user,
                 user_permissions=permissions, config=self.config, env=self.env)
             self.server_impl = server
             return server.run()
 
         else:
             raise Exception('Unrecognised VCS: {}'.format(vcs))
 
     def wrap(self):
         mode = self.mode
         user = self.user
         user_id = self.user_id
         key_id = self.key_id
         shell = self.shell
 
         scm_detected, scm_repo, scm_mode = self.get_repo_details(mode)
 
         log.debug(
             'Mode: `%s` User: `%s:%s` Shell: `%s` SSH Command: `\"%s\"` '
             'SCM_DETECTED: `%s` SCM Mode: `%s` SCM Repo: `%s`',
             mode, user, user_id, shell, self.command,
             scm_detected, scm_mode, scm_repo)
 
         # update last access time for this key
         self.update_key_access_time(key_id)
 
         log.debug('SSH Connection info %s', self.get_connection_info())
 
         if shell and self.command is None:
             log.info(
                 'Dropping to shell, no command given and shell is allowed')
             os.execl('/bin/bash', '-l')
             exit_code = 1
 
         elif scm_detected:
             user = User.get(user_id)
             if not user:
                 log.warning('User with id %s not found', user_id)
                 exit_code = -1
                 return exit_code
 
             auth_user = user.AuthUser()
             permissions = auth_user.permissions['repositories']
 
             try:
                 exit_code, is_updated = self.serve(
                     scm_detected, scm_repo, scm_mode, user, permissions)
             except Exception:
                 log.exception('Error occurred during execution of SshWrapper')
                 exit_code = -1
 
         elif self.command is None and shell is False:
             log.error('No Command given.')
             exit_code = -1
 
         else:
             log.error(
                 'Unhandled Command: "%s" Aborting.', self.command)
             exit_code = -1
 
         return exit_code

rhodecode/apps/ssh_support/tests/conftest.py

@@ -1,62 +1,62
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import pytest
-import ConfigParser
+from pyramid.compat import configparser
 
 from rhodecode.apps.ssh_support.lib.ssh_wrapper import SshWrapper
 from rhodecode.lib.utils2 import AttributeDict
 
 
 @pytest.fixture
 def dummy_conf_file(tmpdir):
-    conf = ConfigParser.ConfigParser()
+    conf = configparser.ConfigParser()
     conf.add_section('app:main')
     conf.set('app:main', 'ssh.executable.hg', '/usr/bin/hg')
     conf.set('app:main', 'ssh.executable.git', '/usr/bin/git')
     conf.set('app:main', 'ssh.executable.svn', '/usr/bin/svnserve')
 
     f_path = os.path.join(str(tmpdir), 'ssh_wrapper_test.ini')
     with open(f_path, 'wb') as f:
         conf.write(f)
 
     return os.path.join(f_path)
 
 
 @pytest.fixture
 def dummy_env():
     return {
         'request':
             AttributeDict(host_url='http://localhost', script_name='/')
     }
 
 
 @pytest.fixture
 def dummy_user():
     return AttributeDict(username='test_user')
 
 
 @pytest.fixture
 def ssh_wrapper(app, dummy_conf_file, dummy_env):
     conn_info = '127.0.0.1 22 10.0.0.1 443'
     return SshWrapper(
         'random command', conn_info, 'auto', 'admin', '1', key_id='1',
         shell=False, ini_path=dummy_conf_file, env=dummy_env)

rhodecode/lib/pyramid_utils.py

@@ -1,49 +1,49
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
-import ConfigParser
-from pyramid.paster import bootstrap as pyramid_bootstrap
+from pyramid.compat import configparser
+from pyramid.paster import bootstrap as pyramid_bootstrap, setup_logging  # noqa
 from pyramid.request import Request
 
 
 def get_config(ini_path, **kwargs):
-    parser = ConfigParser.ConfigParser(**kwargs)
+    parser = configparser.ConfigParser(**kwargs)
     parser.read(ini_path)
     return parser
 
 
 def get_app_config(ini_path):
     from paste.deploy.loadwsgi import appconfig
     return appconfig('config:{}'.format(ini_path), relative_to=os.getcwd())
 
 
 def bootstrap(config_uri, request=None, options=None):
 
     config = get_config(config_uri)
     base_url = 'http://rhodecode.local'
     try:
         base_url = config.get('app:main', 'app.base_url')
-    except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+    except (configparser.NoSectionError, configparser.NoOptionError):
         pass
 
     request = request or Request.blank('/', base_url=base_url)
 
     return pyramid_bootstrap(config_uri, request=request, options=options)

rhodecode/lib/system_info.py

@@ -1,723 +1,722
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2017-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import os
 import sys
 import time
 import platform
 import pkg_resources
 import logging
-import string
 
+from pyramid.compat import configparser
 
 log = logging.getLogger(__name__)
 
 
 psutil = None
 
 try:
     # cygwin cannot have yet psutil support.
     import psutil as psutil
 except ImportError:
     pass
 
 
 _NA = 'NOT AVAILABLE'
 
 STATE_OK = 'ok'
 STATE_ERR = 'error'
 STATE_WARN = 'warning'
 
 STATE_OK_DEFAULT = {'message': '', 'type': STATE_OK}
 
 
 # HELPERS
 def percentage(part, whole):
     whole = float(whole)
     if whole > 0:
         return round(100 * float(part) / whole, 1)
     return 0.0
 
 
 def get_storage_size(storage_path):
     sizes = []
     for file_ in os.listdir(storage_path):
         storage_file = os.path.join(storage_path, file_)
         if os.path.isfile(storage_file):
             try:
                 sizes.append(os.path.getsize(storage_file))
             except OSError:
                 log.exception('Failed to get size of storage file %s',
                               storage_file)
                 pass
 
     return sum(sizes)
 
 
 class SysInfoRes(object):
     def __init__(self, value, state=STATE_OK_DEFAULT, human_value=None):
         self.value = value
         self.state = state
         self.human_value = human_value or value
 
     def __json__(self):
         return {
             'value': self.value,
             'state': self.state,
             'human_value': self.human_value,
         }
 
     def get_value(self):
         return self.__json__()
 
     def __str__(self):
         return '<SysInfoRes({})>'.format(self.__json__())
 
 
 class SysInfo(object):
 
     def __init__(self, func_name, **kwargs):
         self.func_name = func_name
         self.value = _NA
         self.state = None
         self.kwargs = kwargs or {}
 
     def __call__(self):
         computed = self.compute(**self.kwargs)
         if not isinstance(computed, SysInfoRes):
             raise ValueError(
                 'computed value for {} is not instance of '
                 '{}, got {} instead'.format(
                     self.func_name, SysInfoRes, type(computed)))
         return computed.__json__()
 
     def __str__(self):
         return '<SysInfo({})>'.format(self.func_name)
 
     def compute(self, **kwargs):
         return self.func_name(**kwargs)
 
 
 # SysInfo functions
 def python_info():
     value = dict(version=' '.join(platform._sys_version()),
                  executable=sys.executable)
     return SysInfoRes(value=value)
 
 
 def py_modules():
     mods = dict([(p.project_name, p.version)
                  for p in pkg_resources.working_set])
     value = sorted(mods.items(), key=lambda k: k[0].lower())
     return SysInfoRes(value=value)
 
 
 def platform_type():
     from rhodecode.lib.utils import safe_unicode, generate_platform_uuid
 
     value = dict(
         name=safe_unicode(platform.platform()),
         uuid=generate_platform_uuid()
     )
     return SysInfoRes(value=value)
 
 
 def uptime():
     from rhodecode.lib.helpers import age, time_to_datetime
     from rhodecode.translation import TranslationString
 
     value = dict(boot_time=0, uptime=0, text='')
     state = STATE_OK_DEFAULT
     if not psutil:
         return SysInfoRes(value=value, state=state)
 
     boot_time = psutil.boot_time()
     value['boot_time'] = boot_time
     value['uptime'] = time.time() - boot_time
 
     date_or_age = age(time_to_datetime(boot_time))
     if isinstance(date_or_age, TranslationString):
         date_or_age = date_or_age.interpolate()
 
     human_value = value.copy()
     human_value['boot_time'] = time_to_datetime(boot_time)
     human_value['uptime'] = age(time_to_datetime(boot_time), show_suffix=False)
 
     human_value['text'] = u'Server started {}'.format(date_or_age)
     return SysInfoRes(value=value, human_value=human_value)
 
 
 def memory():
     from rhodecode.lib.helpers import format_byte_size_binary
     value = dict(available=0, used=0, used_real=0, cached=0, percent=0,
                  percent_used=0, free=0, inactive=0, active=0, shared=0,
                  total=0, buffers=0, text='')
 
     state = STATE_OK_DEFAULT
     if not psutil:
         return SysInfoRes(value=value, state=state)
 
     value.update(dict(psutil.virtual_memory()._asdict()))
     value['used_real'] = value['total'] - value['available']
     value['percent_used'] = psutil._common.usage_percent(
         value['used_real'], value['total'], 1)
 
     human_value = value.copy()
     human_value['text'] = '%s/%s, %s%% used' % (
         format_byte_size_binary(value['used_real']),
         format_byte_size_binary(value['total']),
         value['percent_used'],)
 
     keys = value.keys()[::]
     keys.pop(keys.index('percent'))
     keys.pop(keys.index('percent_used'))
     keys.pop(keys.index('text'))
     for k in keys:
         human_value[k] = format_byte_size_binary(value[k])
 
     if state['type'] == STATE_OK and value['percent_used'] > 90:
         msg = 'Critical: your available RAM memory is very low.'
         state = {'message': msg, 'type': STATE_ERR}
 
     elif state['type'] == STATE_OK and value['percent_used'] > 70:
         msg = 'Warning: your available RAM memory is running low.'
         state = {'message': msg, 'type': STATE_WARN}
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
 def machine_load():
     value = {'1_min': _NA, '5_min': _NA, '15_min': _NA, 'text': ''}
     state = STATE_OK_DEFAULT
     if not psutil:
         return SysInfoRes(value=value, state=state)
 
     # load averages
     if hasattr(psutil.os, 'getloadavg'):
         value.update(dict(
             zip(['1_min', '5_min', '15_min'], psutil.os.getloadavg())))
 
     human_value = value.copy()
     human_value['text'] = '1min: {}, 5min: {}, 15min: {}'.format(
         value['1_min'], value['5_min'], value['15_min'])
 
     if state['type'] == STATE_OK and value['15_min'] > 5:
         msg = 'Warning: your machine load is very high.'
         state = {'message': msg, 'type': STATE_WARN}
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
 def cpu():
     value = {'cpu': 0, 'cpu_count': 0, 'cpu_usage': []}
     state = STATE_OK_DEFAULT
 
     if not psutil:
         return SysInfoRes(value=value, state=state)
 
     value['cpu'] = psutil.cpu_percent(0.5)
     value['cpu_usage'] = psutil.cpu_percent(0.5, percpu=True)
     value['cpu_count'] = psutil.cpu_count()
 
     human_value = value.copy()
     human_value['text'] = '{} cores at {} %'.format(
         value['cpu_count'], value['cpu'])
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
 def storage():
     from rhodecode.lib.helpers import format_byte_size_binary
     from rhodecode.model.settings import VcsSettingsModel
     path = VcsSettingsModel().get_repos_location()
 
     value = dict(percent=0, used=0, total=0, path=path, text='')
     state = STATE_OK_DEFAULT
     if not psutil:
         return SysInfoRes(value=value, state=state)
 
     try:
         value.update(dict(psutil.disk_usage(path)._asdict()))
     except Exception as e:
         log.exception('Failed to fetch disk info')
         state = {'message': str(e), 'type': STATE_ERR}
 
     human_value = value.copy()
     human_value['used'] = format_byte_size_binary(value['used'])
     human_value['total'] = format_byte_size_binary(value['total'])
     human_value['text'] = "{}/{}, {}% used".format(
         format_byte_size_binary(value['used']),
         format_byte_size_binary(value['total']),
         value['percent'])
 
271