##// END OF EJS Templates
compat: use py3 compatible configparser in all places.
marcink -
r2355:1176c026 default
parent child Browse files
Show More
@@ -1,207 +1,207 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2016-2017 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 import os
22 22 import re
23 23 import logging
24 24 import datetime
25 import ConfigParser
25 from pyramid.compat import configparser
26 26
27 27 from rhodecode.model.db import Session, User, UserSshKeys
28 28 from rhodecode.model.scm import ScmModel
29 29
30 30 from .hg import MercurialServer
31 31 from .git import GitServer
32 32 from .svn import SubversionServer
33 33 log = logging.getLogger(__name__)
34 34
35 35
36 36 class SshWrapper(object):
37 37
38 38 def __init__(self, command, connection_info, mode,
39 39 user, user_id, key_id, shell, ini_path, env):
40 40 self.command = command
41 41 self.connection_info = connection_info
42 42 self.mode = mode
43 43 self.user = user
44 44 self.user_id = user_id
45 45 self.key_id = key_id
46 46 self.shell = shell
47 47 self.ini_path = ini_path
48 48 self.env = env
49 49
50 50 self.config = self.parse_config(ini_path)
51 51 self.server_impl = None
52 52
53 53 def parse_config(self, config_path):
54 parser = ConfigParser.ConfigParser()
54 parser = configparser.ConfigParser()
55 55 parser.read(config_path)
56 56 return parser
57 57
58 58 def update_key_access_time(self, key_id):
59 59 key = UserSshKeys().query().filter(
60 60 UserSshKeys.ssh_key_id == key_id).scalar()
61 61 if key:
62 62 key.accessed_on = datetime.datetime.utcnow()
63 63 Session().add(key)
64 64 Session().commit()
65 65 log.debug('Update key `%s` access time', key_id)
66 66
67 67 def get_connection_info(self):
68 68 """
69 69 connection_info
70 70
71 71 Identifies the client and server ends of the connection.
72 72 The variable contains four space-separated values: client IP address,
73 73 client port number, server IP address, and server port number.
74 74 """
75 75 conn = dict(
76 76 client_ip=None,
77 77 client_port=None,
78 78 server_ip=None,
79 79 server_port=None,
80 80 )
81 81
82 82 info = self.connection_info.split(' ')
83 83 if len(info) == 4:
84 84 conn['client_ip'] = info[0]
85 85 conn['client_port'] = info[1]
86 86 conn['server_ip'] = info[2]
87 87 conn['server_port'] = info[3]
88 88
89 89 return conn
90 90
91 91 def get_repo_details(self, mode):
92 92 vcs_type = mode if mode in ['svn', 'hg', 'git'] else None
93 93 mode = mode
94 94 repo_name = None
95 95
96 96 hg_pattern = r'^hg\s+\-R\s+(\S+)\s+serve\s+\-\-stdio$'
97 97 hg_match = re.match(hg_pattern, self.command)
98 98 if hg_match is not None:
99 99 vcs_type = 'hg'
100 100 repo_name = hg_match.group(1).strip('/')
101 101 return vcs_type, repo_name, mode
102 102
103 103 git_pattern = (
104 104 r'^git-(receive-pack|upload-pack)\s\'[/]?(\S+?)(|\.git)\'$')
105 105 git_match = re.match(git_pattern, self.command)
106 106 if git_match is not None:
107 107 vcs_type = 'git'
108 108 repo_name = git_match.group(2).strip('/')
109 109 mode = git_match.group(1)
110 110 return vcs_type, repo_name, mode
111 111
112 112 svn_pattern = r'^svnserve -t'
113 113 svn_match = re.match(svn_pattern, self.command)
114 114
115 115 if svn_match is not None:
116 116 vcs_type = 'svn'
117 117 # Repo name should be extracted from the input stream
118 118 return vcs_type, repo_name, mode
119 119
120 120 return vcs_type, repo_name, mode
121 121
122 122 def serve(self, vcs, repo, mode, user, permissions):
123 123 store = ScmModel().repos_path
124 124
125 125 log.debug(
126 126 'VCS detected:`%s` mode: `%s` repo_name: %s', vcs, mode, repo)
127 127
128 128 if vcs == 'hg':
129 129 server = MercurialServer(
130 130 store=store, ini_path=self.ini_path,
131 131 repo_name=repo, user=user,
132 132 user_permissions=permissions, config=self.config, env=self.env)
133 133 self.server_impl = server
134 134 return server.run()
135 135
136 136 elif vcs == 'git':
137 137 server = GitServer(
138 138 store=store, ini_path=self.ini_path,
139 139 repo_name=repo, repo_mode=mode, user=user,
140 140 user_permissions=permissions, config=self.config, env=self.env)
141 141 self.server_impl = server
142 142 return server.run()
143 143
144 144 elif vcs == 'svn':
145 145 server = SubversionServer(
146 146 store=store, ini_path=self.ini_path,
147 147 repo_name=None, user=user,
148 148 user_permissions=permissions, config=self.config, env=self.env)
149 149 self.server_impl = server
150 150 return server.run()
151 151
152 152 else:
153 153 raise Exception('Unrecognised VCS: {}'.format(vcs))
154 154
155 155 def wrap(self):
156 156 mode = self.mode
157 157 user = self.user
158 158 user_id = self.user_id
159 159 key_id = self.key_id
160 160 shell = self.shell
161 161
162 162 scm_detected, scm_repo, scm_mode = self.get_repo_details(mode)
163 163
164 164 log.debug(
165 165 'Mode: `%s` User: `%s:%s` Shell: `%s` SSH Command: `\"%s\"` '
166 166 'SCM_DETECTED: `%s` SCM Mode: `%s` SCM Repo: `%s`',
167 167 mode, user, user_id, shell, self.command,
168 168 scm_detected, scm_mode, scm_repo)
169 169
170 170 # update last access time for this key
171 171 self.update_key_access_time(key_id)
172 172
173 173 log.debug('SSH Connection info %s', self.get_connection_info())
174 174
175 175 if shell and self.command is None:
176 176 log.info(
177 177 'Dropping to shell, no command given and shell is allowed')
178 178 os.execl('/bin/bash', '-l')
179 179 exit_code = 1
180 180
181 181 elif scm_detected:
182 182 user = User.get(user_id)
183 183 if not user:
184 184 log.warning('User with id %s not found', user_id)
185 185 exit_code = -1
186 186 return exit_code
187 187
188 188 auth_user = user.AuthUser()
189 189 permissions = auth_user.permissions['repositories']
190 190
191 191 try:
192 192 exit_code, is_updated = self.serve(
193 193 scm_detected, scm_repo, scm_mode, user, permissions)
194 194 except Exception:
195 195 log.exception('Error occurred during execution of SshWrapper')
196 196 exit_code = -1
197 197
198 198 elif self.command is None and shell is False:
199 199 log.error('No Command given.')
200 200 exit_code = -1
201 201
202 202 else:
203 203 log.error(
204 204 'Unhandled Command: "%s" Aborting.', self.command)
205 205 exit_code = -1
206 206
207 207 return exit_code
@@ -1,62 +1,62 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2016-2017 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 import os
22 22 import pytest
23 import ConfigParser
23 from pyramid.compat import configparser
24 24
25 25 from rhodecode.apps.ssh_support.lib.ssh_wrapper import SshWrapper
26 26 from rhodecode.lib.utils2 import AttributeDict
27 27
28 28
29 29 @pytest.fixture
30 30 def dummy_conf_file(tmpdir):
31 conf = ConfigParser.ConfigParser()
31 conf = configparser.ConfigParser()
32 32 conf.add_section('app:main')
33 33 conf.set('app:main', 'ssh.executable.hg', '/usr/bin/hg')
34 34 conf.set('app:main', 'ssh.executable.git', '/usr/bin/git')
35 35 conf.set('app:main', 'ssh.executable.svn', '/usr/bin/svnserve')
36 36
37 37 f_path = os.path.join(str(tmpdir), 'ssh_wrapper_test.ini')
38 38 with open(f_path, 'wb') as f:
39 39 conf.write(f)
40 40
41 41 return os.path.join(f_path)
42 42
43 43
44 44 @pytest.fixture
45 45 def dummy_env():
46 46 return {
47 47 'request':
48 48 AttributeDict(host_url='http://localhost', script_name='/')
49 49 }
50 50
51 51
52 52 @pytest.fixture
53 53 def dummy_user():
54 54 return AttributeDict(username='test_user')
55 55
56 56
57 57 @pytest.fixture
58 58 def ssh_wrapper(app, dummy_conf_file, dummy_env):
59 59 conn_info = '127.0.0.1 22 10.0.0.1 443'
60 60 return SshWrapper(
61 61 'random command', conn_info, 'auto', 'admin', '1', key_id='1',
62 62 shell=False, ini_path=dummy_conf_file, env=dummy_env)
@@ -1,49 +1,49 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2016-2017 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 import os
22 import ConfigParser
23 from pyramid.paster import bootstrap as pyramid_bootstrap
22 from pyramid.compat import configparser
23 from pyramid.paster import bootstrap as pyramid_bootstrap, setup_logging # noqa
24 24 from pyramid.request import Request
25 25
26 26
27 27 def get_config(ini_path, **kwargs):
28 parser = ConfigParser.ConfigParser(**kwargs)
28 parser = configparser.ConfigParser(**kwargs)
29 29 parser.read(ini_path)
30 30 return parser
31 31
32 32
33 33 def get_app_config(ini_path):
34 34 from paste.deploy.loadwsgi import appconfig
35 35 return appconfig('config:{}'.format(ini_path), relative_to=os.getcwd())
36 36
37 37
38 38 def bootstrap(config_uri, request=None, options=None):
39 39
40 40 config = get_config(config_uri)
41 41 base_url = 'http://rhodecode.local'
42 42 try:
43 43 base_url = config.get('app:main', 'app.base_url')
44 except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
44 except (configparser.NoSectionError, configparser.NoOptionError):
45 45 pass
46 46
47 47 request = request or Request.blank('/', base_url=base_url)
48 48
49 49 return pyramid_bootstrap(config_uri, request=request, options=options)
@@ -1,723 +1,722 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2017-2017 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21
22 22 import os
23 23 import sys
24 24 import time
25 25 import platform
26 26 import pkg_resources
27 27 import logging
28 import string
29 28
29 from pyramid.compat import configparser
30 30
31 31 log = logging.getLogger(__name__)
32 32
33 33
34 34 psutil = None
35 35
36 36 try:
37 37 # cygwin cannot have yet psutil support.
38 38 import psutil as psutil
39 39 except ImportError:
40 40 pass
41 41
42 42
43 43 _NA = 'NOT AVAILABLE'
44 44
45 45 STATE_OK = 'ok'
46 46 STATE_ERR = 'error'
47 47 STATE_WARN = 'warning'
48 48
49 49 STATE_OK_DEFAULT = {'message': '', 'type': STATE_OK}
50 50
51 51
52 52 # HELPERS
53 53 def percentage(part, whole):
54 54 whole = float(whole)
55 55 if whole > 0:
56 56 return round(100 * float(part) / whole, 1)
57 57 return 0.0
58 58
59 59
60 60 def get_storage_size(storage_path):
61 61 sizes = []
62 62 for file_ in os.listdir(storage_path):
63 63 storage_file = os.path.join(storage_path, file_)
64 64 if os.path.isfile(storage_file):
65 65 try:
66 66 sizes.append(os.path.getsize(storage_file))
67 67 except OSError:
68 68 log.exception('Failed to get size of storage file %s',
69 69 storage_file)
70 70 pass
71 71
72 72 return sum(sizes)
73 73
74 74
75 75 class SysInfoRes(object):
76 76 def __init__(self, value, state=STATE_OK_DEFAULT, human_value=None):
77 77 self.value = value
78 78 self.state = state
79 79 self.human_value = human_value or value
80 80
81 81 def __json__(self):
82 82 return {
83 83 'value': self.value,
84 84 'state': self.state,
85 85 'human_value': self.human_value,
86 86 }
87 87
88 88 def get_value(self):
89 89 return self.__json__()
90 90
91 91 def __str__(self):
92 92 return '<SysInfoRes({})>'.format(self.__json__())
93 93
94 94
95 95 class SysInfo(object):
96 96
97 97 def __init__(self, func_name, **kwargs):
98 98 self.func_name = func_name
99 99 self.value = _NA
100 100 self.state = None
101 101 self.kwargs = kwargs or {}
102 102
103 103 def __call__(self):
104 104 computed = self.compute(**self.kwargs)
105 105 if not isinstance(computed, SysInfoRes):
106 106 raise ValueError(
107 107 'computed value for {} is not instance of '
108 108 '{}, got {} instead'.format(
109 109 self.func_name, SysInfoRes, type(computed)))
110 110 return computed.__json__()
111 111
112 112 def __str__(self):
113 113 return '<SysInfo({})>'.format(self.func_name)
114 114
115 115 def compute(self, **kwargs):
116 116 return self.func_name(**kwargs)
117 117
118 118
119 119 # SysInfo functions
120 120 def python_info():
121 121 value = dict(version=' '.join(platform._sys_version()),
122 122 executable=sys.executable)
123 123 return SysInfoRes(value=value)
124 124
125 125
126 126 def py_modules():
127 127 mods = dict([(p.project_name, p.version)
128 128 for p in pkg_resources.working_set])
129 129 value = sorted(mods.items(), key=lambda k: k[0].lower())
130 130 return SysInfoRes(value=value)
131 131
132 132
133 133 def platform_type():
134 134 from rhodecode.lib.utils import safe_unicode, generate_platform_uuid
135 135
136 136 value = dict(
137 137 name=safe_unicode(platform.platform()),
138 138 uuid=generate_platform_uuid()
139 139 )
140 140 return SysInfoRes(value=value)
141 141
142 142
143 143 def uptime():
144 144 from rhodecode.lib.helpers import age, time_to_datetime
145 145 from rhodecode.translation import TranslationString
146 146
147 147 value = dict(boot_time=0, uptime=0, text='')
148 148 state = STATE_OK_DEFAULT
149 149 if not psutil:
150 150 return SysInfoRes(value=value, state=state)
151 151
152 152 boot_time = psutil.boot_time()
153 153 value['boot_time'] = boot_time
154 154 value['uptime'] = time.time() - boot_time
155 155
156 156 date_or_age = age(time_to_datetime(boot_time))
157 157 if isinstance(date_or_age, TranslationString):
158 158 date_or_age = date_or_age.interpolate()
159 159
160 160 human_value = value.copy()
161 161 human_value['boot_time'] = time_to_datetime(boot_time)
162 162 human_value['uptime'] = age(time_to_datetime(boot_time), show_suffix=False)
163 163
164 164 human_value['text'] = u'Server started {}'.format(date_or_age)
165 165 return SysInfoRes(value=value, human_value=human_value)
166 166
167 167
168 168 def memory():
169 169 from rhodecode.lib.helpers import format_byte_size_binary
170 170 value = dict(available=0, used=0, used_real=0, cached=0, percent=0,
171 171 percent_used=0, free=0, inactive=0, active=0, shared=0,
172 172 total=0, buffers=0, text='')
173 173
174 174 state = STATE_OK_DEFAULT
175 175 if not psutil:
176 176 return SysInfoRes(value=value, state=state)
177 177
178 178 value.update(dict(psutil.virtual_memory()._asdict()))
179 179 value['used_real'] = value['total'] - value['available']
180 180 value['percent_used'] = psutil._common.usage_percent(
181 181 value['used_real'], value['total'], 1)
182 182
183 183 human_value = value.copy()
184 184 human_value['text'] = '%s/%s, %s%% used' % (
185 185 format_byte_size_binary(value['used_real']),
186 186 format_byte_size_binary(value['total']),
187 187 value['percent_used'],)
188 188
189 189 keys = value.keys()[::]
190 190 keys.pop(keys.index('percent'))
191 191 keys.pop(keys.index('percent_used'))
192 192 keys.pop(keys.index('text'))
193 193 for k in keys:
194 194 human_value[k] = format_byte_size_binary(value[k])
195 195
196 196 if state['type'] == STATE_OK and value['percent_used'] > 90:
197 197 msg = 'Critical: your available RAM memory is very low.'
198 198 state = {'message': msg, 'type': STATE_ERR}
199 199
200 200 elif state['type'] == STATE_OK and value['percent_used'] > 70:
201 201 msg = 'Warning: your available RAM memory is running low.'
202 202 state = {'message': msg, 'type': STATE_WARN}
203 203
204 204 return SysInfoRes(value=value, state=state, human_value=human_value)
205 205
206 206
207 207 def machine_load():
208 208 value = {'1_min': _NA, '5_min': _NA, '15_min': _NA, 'text': ''}
209 209 state = STATE_OK_DEFAULT
210 210 if not psutil:
211 211 return SysInfoRes(value=value, state=state)
212 212
213 213 # load averages
214 214 if hasattr(psutil.os, 'getloadavg'):
215 215 value.update(dict(
216 216 zip(['1_min', '5_min', '15_min'], psutil.os.getloadavg())))
217 217
218 218 human_value = value.copy()
219 219 human_value['text'] = '1min: {}, 5min: {}, 15min: {}'.format(
220 220 value['1_min'], value['5_min'], value['15_min'])
221 221
222 222 if state['type'] == STATE_OK and value['15_min'] > 5:
223 223 msg = 'Warning: your machine load is very high.'
224 224 state = {'message': msg, 'type': STATE_WARN}
225 225
226 226 return SysInfoRes(value=value, state=state, human_value=human_value)
227 227
228 228
229 229 def cpu():
230 230 value = {'cpu': 0, 'cpu_count': 0, 'cpu_usage': []}
231 231 state = STATE_OK_DEFAULT
232 232
233 233 if not psutil:
234 234 return SysInfoRes(value=value, state=state)
235 235
236 236 value['cpu'] = psutil.cpu_percent(0.5)
237 237 value['cpu_usage'] = psutil.cpu_percent(0.5, percpu=True)
238 238 value['cpu_count'] = psutil.cpu_count()
239 239
240 240 human_value = value.copy()
241 241 human_value['text'] = '{} cores at {} %'.format(
242 242 value['cpu_count'], value['cpu'])
243 243
244 244 return SysInfoRes(value=value, state=state, human_value=human_value)
245 245
246 246
247 247 def storage():
248 248 from rhodecode.lib.helpers import format_byte_size_binary
249 249 from rhodecode.model.settings import VcsSettingsModel
250 250 path = VcsSettingsModel().get_repos_location()
251 251
252 252 value = dict(percent=0, used=0, total=0, path=path, text='')
253 253 state = STATE_OK_DEFAULT
254 254 if not psutil:
255 255 return SysInfoRes(value=value, state=state)
256 256
257 257 try:
258 258 value.update(dict(psutil.disk_usage(path)._asdict()))
259 259 except Exception as e:
260 260 log.exception('Failed to fetch disk info')
261 261 state = {'message': str(e), 'type': STATE_ERR}
262 262
263 263 human_value = value.copy()
264 264 human_value['used'] = format_byte_size_binary(value['used'])
265 265 human_value['total'] = format_byte_size_binary(value['total'])
266 266 human_value['text'] = "{}/{}, {}% used".format(
267 267 format_byte_size_binary(value['used']),
268 268 format_byte_size_binary(value['total']),
269 269 value['percent'])
270 270
271 271 if state['type'] == STATE_OK and value['percent'] > 90:
272 272 msg = 'Critical: your disk space is very low.'
273 273 state = {'message': msg, 'type': STATE_ERR}
274 274
275 275 elif state['type'] == STATE_OK and value['percent'] > 70:
276 276 msg = 'Warning: your disk space is running low.'
277 277 state = {'message': msg, 'type': STATE_WARN}
278 278
279 279 return SysInfoRes(value=value, state=state, human_value=human_value)
280 280
281 281
282 282 def storage_inodes():
283 283 from rhodecode.model.settings import VcsSettingsModel
284 284 path = VcsSettingsModel().get_repos_location()
285 285
286 286 value = dict(percent=0, free=0, used=0, total=0, path=path, text='')
287 287 state = STATE_OK_DEFAULT
288 288 if not psutil:
289 289 return SysInfoRes(value=value, state=state)
290 290
291 291 try:
292 292 i_stat = os.statvfs(path)
293 293 value['free'] = i_stat.f_ffree
294 294 value['used'] = i_stat.f_files-i_stat.f_favail
295 295 value['total'] = i_stat.f_files
296 296 value['percent'] = percentage(value['used'], value['total'])
297 297 except Exception as e:
298 298 log.exception('Failed to fetch disk inodes info')
299 299 state = {'message': str(e), 'type': STATE_ERR}
300 300
301 301 human_value = value.copy()
302 302 human_value['text'] = "{}/{}, {}% used".format(
303 303 value['used'], value['total'], value['percent'])
304 304
305 305 if state['type'] == STATE_OK and value['percent'] > 90:
306 306 msg = 'Critical: your disk free inodes are very low.'
307 307 state = {'message': msg, 'type': STATE_ERR}
308 308
309 309 elif state['type'] == STATE_OK and value['percent'] > 70:
310 310 msg = 'Warning: your disk free inodes are running low.'
311 311 state = {'message': msg, 'type': STATE_WARN}
312 312
313 313 return SysInfoRes(value=value, state=state, human_value=human_value)
314 314
315 315
316 316 def storage_archives():
317 317 import rhodecode
318 318 from rhodecode.lib.utils import safe_str
319 319 from rhodecode.lib.helpers import format_byte_size_binary
320 320
321 321 msg = 'Enable this by setting ' \
322 322 'archive_cache_dir=/path/to/cache option in the .ini file'
323 323 path = safe_str(rhodecode.CONFIG.get('archive_cache_dir', msg))
324 324
325 325 value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
326 326 state = STATE_OK_DEFAULT
327 327 try:
328 328 items_count = 0
329 329 used = 0
330 330 for root, dirs, files in os.walk(path):
331 331 if root == path:
332 332 items_count = len(files)
333 333
334 334 for f in files:
335 335 try:
336 336 used += os.path.getsize(os.path.join(root, f))
337 337 except OSError:
338 338 pass
339 339 value.update({
340 340 'percent': 100,
341 341 'used': used,
342 342 'total': used,
343 343 'items': items_count
344 344 })
345 345
346 346 except Exception as e:
347 347 log.exception('failed to fetch archive cache storage')
348 348 state = {'message': str(e), 'type': STATE_ERR}
349 349
350 350 human_value = value.copy()
351 351 human_value['used'] = format_byte_size_binary(value['used'])
352 352 human_value['total'] = format_byte_size_binary(value['total'])
353 353 human_value['text'] = "{} ({} items)".format(
354 354 human_value['used'], value['items'])
355 355
356 356 return SysInfoRes(value=value, state=state, human_value=human_value)
357 357
358 358
359 359 def storage_gist():
360 360 from rhodecode.model.gist import GIST_STORE_LOC
361 361 from rhodecode.model.settings import VcsSettingsModel
362 362 from rhodecode.lib.utils import safe_str
363 363 from rhodecode.lib.helpers import format_byte_size_binary
364 364 path = safe_str(os.path.join(
365 365 VcsSettingsModel().get_repos_location(), GIST_STORE_LOC))
366 366
367 367 # gist storage
368 368 value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
369 369 state = STATE_OK_DEFAULT
370 370
371 371 try:
372 372 items_count = 0
373 373 used = 0
374 374 for root, dirs, files in os.walk(path):
375 375 if root == path:
376 376 items_count = len(dirs)
377 377
378 378 for f in files:
379 379 try:
380 380 used += os.path.getsize(os.path.join(root, f))
381 381 except OSError:
382 382 pass
383 383 value.update({
384 384 'percent': 100,
385 385 'used': used,
386 386 'total': used,
387 387 'items': items_count
388 388 })
389 389 except Exception as e:
390 390 log.exception('failed to fetch gist storage items')
391 391 state = {'message': str(e), 'type': STATE_ERR}
392 392
393 393 human_value = value.copy()
394 394 human_value['used'] = format_byte_size_binary(value['used'])
395 395 human_value['total'] = format_byte_size_binary(value['total'])
396 396 human_value['text'] = "{} ({} items)".format(
397 397 human_value['used'], value['items'])
398 398
399 399 return SysInfoRes(value=value, state=state, human_value=human_value)
400 400
401 401
402 402 def storage_temp():
403 403 import tempfile
404 404 from rhodecode.lib.helpers import format_byte_size_binary
405 405
406 406 path = tempfile.gettempdir()
407 407 value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
408 408 state = STATE_OK_DEFAULT
409 409
410 410 if not psutil:
411 411 return SysInfoRes(value=value, state=state)
412 412
413 413 try:
414 414 value.update(dict(psutil.disk_usage(path)._asdict()))
415 415 except Exception as e:
416 416 log.exception('Failed to fetch temp dir info')
417 417 state = {'message': str(e), 'type': STATE_ERR}
418 418
419 419 human_value = value.copy()
420 420 human_value['used'] = format_byte_size_binary(value['used'])
421 421 human_value['total'] = format_byte_size_binary(value['total'])
422 422 human_value['text'] = "{}/{}, {}% used".format(
423 423 format_byte_size_binary(value['used']),
424 424 format_byte_size_binary(value['total']),
425 425 value['percent'])
426 426
427 427 return SysInfoRes(value=value, state=state, human_value=human_value)
428 428
429 429
430 430 def search_info():
431 431 import rhodecode
432 432 from rhodecode.lib.index import searcher_from_config
433 433
434 434 backend = rhodecode.CONFIG.get('search.module', '')
435 435 location = rhodecode.CONFIG.get('search.location', '')
436 436
437 437 try:
438 438 searcher = searcher_from_config(rhodecode.CONFIG)
439 439 searcher = searcher.__class__.__name__
440 440 except Exception:
441 441 searcher = None
442 442
443 443 value = dict(
444 444 backend=backend, searcher=searcher, location=location, text='')
445 445 state = STATE_OK_DEFAULT
446 446
447 447 human_value = value.copy()
448 448 human_value['text'] = "backend:`{}`".format(human_value['backend'])
449 449
450 450 return SysInfoRes(value=value, state=state, human_value=human_value)
451 451
452 452
453 453 def git_info():
454 454 from rhodecode.lib.vcs.backends import git
455 455 state = STATE_OK_DEFAULT
456 456 value = human_value = ''
457 457 try:
458 458 value = git.discover_git_version(raise_on_exc=True)
459 459 human_value = 'version reported from VCSServer: {}'.format(value)
460 460 except Exception as e:
461 461 state = {'message': str(e), 'type': STATE_ERR}
462 462
463 463 return SysInfoRes(value=value, state=state, human_value=human_value)
464 464
465 465
466 466 def hg_info():
467 467 from rhodecode.lib.vcs.backends import hg
468 468 state = STATE_OK_DEFAULT
469 469 value = human_value = ''
470 470 try:
471 471 value = hg.discover_hg_version(raise_on_exc=True)
472 472 human_value = 'version reported from VCSServer: {}'.format(value)
473 473 except Exception as e:
474 474 state = {'message': str(e), 'type': STATE_ERR}
475 475 return SysInfoRes(value=value, state=state, human_value=human_value)
476 476
477 477
478 478 def svn_info():
479 479 from rhodecode.lib.vcs.backends import svn
480 480 state = STATE_OK_DEFAULT
481 481 value = human_value = ''
482 482 try:
483 483 value = svn.discover_svn_version(raise_on_exc=True)
484 484 human_value = 'version reported from VCSServer: {}'.format(value)
485 485 except Exception as e:
486 486 state = {'message': str(e), 'type': STATE_ERR}
487 487 return SysInfoRes(value=value, state=state, human_value=human_value)
488 488
489 489
490 490 def vcs_backends():
491 491 import rhodecode
492 492 value = rhodecode.CONFIG.get('vcs.backends')
493 493 human_value = 'Enabled backends in order: {}'.format(','.join(value))
494 494 return SysInfoRes(value=value, human_value=human_value)
495 495
496 496
497 497 def vcs_server():
498 498 import rhodecode
499 499 from rhodecode.lib.vcs.backends import get_vcsserver_service_data
500 500
501 501 server_url = rhodecode.CONFIG.get('vcs.server')
502 502 enabled = rhodecode.CONFIG.get('vcs.server.enable')
503 503 protocol = rhodecode.CONFIG.get('vcs.server.protocol') or 'http'
504 504 state = STATE_OK_DEFAULT
505 505 version = None
506 506 workers = 0
507 507
508 508 try:
509 509 data = get_vcsserver_service_data()
510 510 if data and 'version' in data:
511 511 version = data['version']
512 512
513 513 if data and 'config' in data:
514 514 conf = data['config']
515 515 workers = conf.get('workers', 'NOT AVAILABLE')
516 516
517 517 connection = 'connected'
518 518 except Exception as e:
519 519 connection = 'failed'
520 520 state = {'message': str(e), 'type': STATE_ERR}
521 521
522 522 value = dict(
523 523 url=server_url,
524 524 enabled=enabled,
525 525 protocol=protocol,
526 526 connection=connection,
527 527 version=version,
528 528 text='',
529 529 )
530 530
531 531 human_value = value.copy()
532 532 human_value['text'] = \
533 533 '{url}@ver:{ver} via {mode} mode[workers:{workers}], connection:{conn}'.format(
534 534 url=server_url, ver=version, workers=workers, mode=protocol,
535 535 conn=connection)
536 536
537 537 return SysInfoRes(value=value, state=state, human_value=human_value)
538 538
539 539
540 540 def rhodecode_app_info():
541 541 import rhodecode
542 542 edition = rhodecode.CONFIG.get('rhodecode.edition')
543 543
544 544 value = dict(
545 545 rhodecode_version=rhodecode.__version__,
546 546 rhodecode_lib_path=os.path.abspath(rhodecode.__file__),
547 547 text=''
548 548 )
549 549 human_value = value.copy()
550 550 human_value['text'] = 'RhodeCode {edition}, version {ver}'.format(
551 551 edition=edition, ver=value['rhodecode_version']
552 552 )
553 553 return SysInfoRes(value=value, human_value=human_value)
554 554
555 555
556 556 def rhodecode_config():
557 557 import rhodecode
558 import ConfigParser as configparser
559 558 path = rhodecode.CONFIG.get('__file__')
560 559 rhodecode_ini_safe = rhodecode.CONFIG.copy()
561 560
562 561 try:
563 562 config = configparser.ConfigParser()
564 563 config.read(path)
565 564 parsed_ini = config
566 565 if parsed_ini.has_section('server:main'):
567 566 parsed_ini = dict(parsed_ini.items('server:main'))
568 567 except Exception:
569 568 log.exception('Failed to read .ini file for display')
570 569 parsed_ini = {}
571 570
572 571 cert_path = os.path.join(
573 572 os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(path)))),
574 573 '.rccontrol-profile/etc/ca-bundle.crt')
575 574
576 575 rhodecode_ini_safe['server:main'] = parsed_ini
577 576
578 577 blacklist = [
579 578 'rhodecode_license_key',
580 579 'routes.map',
581 580 'sqlalchemy.db1.url',
582 581 'channelstream.secret',
583 582 'beaker.session.secret',
584 583 'rhodecode.encrypted_values.secret',
585 584 'rhodecode_auth_github_consumer_key',
586 585 'rhodecode_auth_github_consumer_secret',
587 586 'rhodecode_auth_google_consumer_key',
588 587 'rhodecode_auth_google_consumer_secret',
589 588 'rhodecode_auth_bitbucket_consumer_secret',
590 589 'rhodecode_auth_bitbucket_consumer_key',
591 590 'rhodecode_auth_twitter_consumer_secret',
592 591 'rhodecode_auth_twitter_consumer_key',
593 592
594 593 'rhodecode_auth_twitter_secret',
595 594 'rhodecode_auth_github_secret',
596 595 'rhodecode_auth_google_secret',
597 596 'rhodecode_auth_bitbucket_secret',
598 597
599 598 'appenlight.api_key',
600 599 ('app_conf', 'sqlalchemy.db1.url')
601 600 ]
602 601 for k in blacklist:
603 602 if isinstance(k, tuple):
604 603 section, key = k
605 604 if section in rhodecode_ini_safe:
606 605 rhodecode_ini_safe[section] = '**OBFUSCATED**'
607 606 else:
608 607 rhodecode_ini_safe.pop(k, None)
609 608
610 609 # TODO: maybe put some CONFIG checks here ?
611 610 return SysInfoRes(value={'config': rhodecode_ini_safe,
612 611 'path': path, 'cert_path': cert_path})
613 612
614 613
615 614 def database_info():
616 615 import rhodecode
617 616 from sqlalchemy.engine import url as engine_url
618 617 from rhodecode.model.meta import Base as sql_base, Session
619 618 from rhodecode.model.db import DbMigrateVersion
620 619
621 620 state = STATE_OK_DEFAULT
622 621
623 622 db_migrate = DbMigrateVersion.query().filter(
624 623 DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one()
625 624
626 625 db_url_obj = engine_url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url'])
627 626
628 627 try:
629 628 engine = sql_base.metadata.bind
630 629 db_server_info = engine.dialect._get_server_version_info(
631 630 Session.connection(bind=engine))
632 631 db_version = '.'.join(map(str, db_server_info))
633 632 except Exception:
634 633 log.exception('failed to fetch db version')
635 634 db_version = 'UNKNOWN'
636 635
637 636 db_info = dict(
638 637 migrate_version=db_migrate.version,
639 638 type=db_url_obj.get_backend_name(),
640 639 version=db_version,
641 640 url=repr(db_url_obj)
642 641 )
643 642 current_version = db_migrate.version
644 643 expected_version = rhodecode.__dbversion__
645 644 if state['type'] == STATE_OK and current_version != expected_version:
646 645 msg = 'Critical: database schema mismatch, ' \
647 646 'expected version {}, got {}. ' \
648 647 'Please run migrations on your database.'.format(
649 648 expected_version, current_version)
650 649 state = {'message': msg, 'type': STATE_ERR}
651 650
652 651 human_value = db_info.copy()
653 652 human_value['url'] = "{} @ migration version: {}".format(
654 653 db_info['url'], db_info['migrate_version'])
655 654 human_value['version'] = "{} {}".format(db_info['type'], db_info['version'])
656 655 return SysInfoRes(value=db_info, state=state, human_value=human_value)
657 656
658 657
659 658 def server_info(environ):
660 659 import rhodecode
661 660 from rhodecode.lib.base import get_server_ip_addr, get_server_port
662 661
663 662 value = {
664 663 'server_ip': '%s:%s' % (
665 664 get_server_ip_addr(environ, log_errors=False),
666 665 get_server_port(environ)
667 666 ),
668 667 'server_id': rhodecode.CONFIG.get('instance_id'),
669 668 }
670 669 return SysInfoRes(value=value)
671 670
672 671
673 672 def usage_info():
674 673 from rhodecode.model.db import User, Repository
675 674 value = {
676 675 'users': User.query().count(),
677 676 'users_active': User.query().filter(User.active == True).count(),
678 677 'repositories': Repository.query().count(),
679 678 'repository_types': {
680 679 'hg': Repository.query().filter(
681 680 Repository.repo_type == 'hg').count(),
682 681 'git': Repository.query().filter(
683 682 Repository.repo_type == 'git').count(),
684 683 'svn': Repository.query().filter(
685 684 Repository.repo_type == 'svn').count(),
686 685 },
687 686 }
688 687 return SysInfoRes(value=value)
689 688
690 689
691 690 def get_system_info(environ):
692 691 environ = environ or {}
693 692 return {
694 693 'rhodecode_app': SysInfo(rhodecode_app_info)(),
695 694 'rhodecode_config': SysInfo(rhodecode_config)(),
696 695 'rhodecode_usage': SysInfo(usage_info)(),
697 696 'python': SysInfo(python_info)(),
698 697 'py_modules': SysInfo(py_modules)(),
699 698
700 699 'platform': SysInfo(platform_type)(),
701 700 'server': SysInfo(server_info, environ=environ)(),
702 701 'database': SysInfo(database_info)(),
703 702
704 703 'storage': SysInfo(storage)(),
705 704 'storage_inodes': SysInfo(storage_inodes)(),
706 705 'storage_archive': SysInfo(storage_archives)(),
707 706 'storage_gist': SysInfo(storage_gist)(),
708 707 'storage_temp': SysInfo(storage_temp)(),
709 708
710 709 'search': SysInfo(search_info)(),
711 710
712 711 'uptime': SysInfo(uptime)(),
713 712 'load': SysInfo(machine_load)(),
714 713 'cpu': SysInfo(cpu)(),
715 714 'memory': SysInfo(memory)(),
716 715
717 716 'vcs_backends': SysInfo(vcs_backends)(),
718 717 'vcs_server': SysInfo(vcs_server)(),
719 718
720 719 'git': SysInfo(git_info)(),
721 720 'hg': SysInfo(hg_info)(),
722 721 'svn': SysInfo(svn_info)(),
723 722 }
@@ -1,545 +1,545 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2014-2017 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 """
22 22 GIT commit module
23 23 """
24 24
25 25 import re
26 26 import stat
27 from ConfigParser import ConfigParser
28 27 from itertools import chain
29 28 from StringIO import StringIO
30 29
31 30 from zope.cachedescriptors.property import Lazy as LazyProperty
31 from pyramid.compat import configparser
32 32
33 33 from rhodecode.lib.datelib import utcdate_fromtimestamp
34 34 from rhodecode.lib.utils import safe_unicode, safe_str
35 35 from rhodecode.lib.utils2 import safe_int
36 36 from rhodecode.lib.vcs.conf import settings
37 37 from rhodecode.lib.vcs.backends import base
38 38 from rhodecode.lib.vcs.exceptions import CommitError, NodeDoesNotExistError
39 39 from rhodecode.lib.vcs.nodes import (
40 40 FileNode, DirNode, NodeKind, RootNode, SubModuleNode,
41 41 ChangedFileNodesGenerator, AddedFileNodesGenerator,
42 42 RemovedFileNodesGenerator, LargeFileNode)
43 43
44 44
45 45 class GitCommit(base.BaseCommit):
46 46 """
47 47 Represents state of the repository at single commit id.
48 48 """
49 49 _author_property = 'author'
50 50 _committer_property = 'committer'
51 51 _date_property = 'commit_time'
52 52 _date_tz_property = 'commit_timezone'
53 53 _message_property = 'message'
54 54 _parents_property = 'parents'
55 55
56 56 _filter_pre_load = [
57 57 # done through a more complex tree walk on parents
58 58 "affected_files",
59 59 # based on repository cached property
60 60 "branch",
61 61 # done through subprocess not remote call
62 62 "children",
63 63 # done through a more complex tree walk on parents
64 64 "status",
65 65 # mercurial specific property not supported here
66 66 "_file_paths",
67 67 # mercurial specific property not supported here
68 68 'obsolete',
69 69 # mercurial specific property not supported here
70 70 'phase',
71 71 # mercurial specific property not supported here
72 72 'hidden'
73 73 ]
74 74
75 75 def __init__(self, repository, raw_id, idx, pre_load=None):
76 76 self.repository = repository
77 77 self._remote = repository._remote
78 78 # TODO: johbo: Tweak of raw_id should not be necessary
79 79 self.raw_id = safe_str(raw_id)
80 80 self.idx = idx
81 81
82 82 self._set_bulk_properties(pre_load)
83 83
84 84 # caches
85 85 self._stat_modes = {} # stat info for paths
86 86 self._paths = {} # path processed with parse_tree
87 87 self.nodes = {}
88 88 self._submodules = None
89 89
90 90 def _set_bulk_properties(self, pre_load):
91 91 if not pre_load:
92 92 return
93 93 pre_load = [entry for entry in pre_load
94 94 if entry not in self._filter_pre_load]
95 95 if not pre_load:
96 96 return
97 97
98 98 result = self._remote.bulk_request(self.raw_id, pre_load)
99 99 for attr, value in result.items():
100 100 if attr in ["author", "message"]:
101 101 if value:
102 102 value = safe_unicode(value)
103 103 elif attr == "date":
104 104 value = utcdate_fromtimestamp(*value)
105 105 elif attr == "parents":
106 106 value = self._make_commits(value)
107 107 self.__dict__[attr] = value
108 108
109 109 @LazyProperty
110 110 def _commit(self):
111 111 return self._remote[self.raw_id]
112 112
113 113 @LazyProperty
114 114 def _tree_id(self):
115 115 return self._remote[self._commit['tree']]['id']
116 116
117 117 @LazyProperty
118 118 def id(self):
119 119 return self.raw_id
120 120
121 121 @LazyProperty
122 122 def short_id(self):
123 123 return self.raw_id[:12]
124 124
125 125 @LazyProperty
126 126 def message(self):
127 127 return safe_unicode(
128 128 self._remote.commit_attribute(self.id, self._message_property))
129 129
130 130 @LazyProperty
131 131 def committer(self):
132 132 return safe_unicode(
133 133 self._remote.commit_attribute(self.id, self._committer_property))
134 134
135 135 @LazyProperty
136 136 def author(self):
137 137 return safe_unicode(
138 138 self._remote.commit_attribute(self.id, self._author_property))
139 139
140 140 @LazyProperty
141 141 def date(self):
142 142 unix_ts, tz = self._remote.get_object_attrs(
143 143 self.raw_id, self._date_property, self._date_tz_property)
144 144 return utcdate_fromtimestamp(unix_ts, tz)
145 145
146 146 @LazyProperty
147 147 def status(self):
148 148 """
149 149 Returns modified, added, removed, deleted files for current commit
150 150 """
151 151 return self.changed, self.added, self.removed
152 152
153 153 @LazyProperty
154 154 def tags(self):
155 155 tags = [safe_unicode(name) for name,
156 156 commit_id in self.repository.tags.iteritems()
157 157 if commit_id == self.raw_id]
158 158 return tags
159 159
160 160 @LazyProperty
161 161 def branch(self):
162 162 for name, commit_id in self.repository.branches.iteritems():
163 163 if commit_id == self.raw_id:
164 164 return safe_unicode(name)
165 165 return None
166 166
167 167 def _get_id_for_path(self, path):
168 168 path = safe_str(path)
169 169 if path in self._paths:
170 170 return self._paths[path]
171 171
172 172 tree_id = self._tree_id
173 173
174 174 path = path.strip('/')
175 175 if path == '':
176 176 data = [tree_id, "tree"]
177 177 self._paths[''] = data
178 178 return data
179 179
180 180 parts = path.split('/')
181 181 dirs, name = parts[:-1], parts[-1]
182 182 cur_dir = ''
183 183
184 184 # initially extract things from root dir
185 185 tree_items = self._remote.tree_items(tree_id)
186 186 self._process_tree_items(tree_items, cur_dir)
187 187
188 188 for dir in dirs:
189 189 if cur_dir:
190 190 cur_dir = '/'.join((cur_dir, dir))
191 191 else:
192 192 cur_dir = dir
193 193 dir_id = None
194 194 for item, stat_, id_, type_ in tree_items:
195 195 if item == dir:
196 196 dir_id = id_
197 197 break
198 198 if dir_id:
199 199 if type_ != "tree":
200 200 raise CommitError('%s is not a directory' % cur_dir)
201 201 # update tree
202 202 tree_items = self._remote.tree_items(dir_id)
203 203 else:
204 204 raise CommitError('%s have not been found' % cur_dir)
205 205
206 206 # cache all items from the given traversed tree
207 207 self._process_tree_items(tree_items, cur_dir)
208 208
209 209 if path not in self._paths:
210 210 raise self.no_node_at_path(path)
211 211
212 212 return self._paths[path]
213 213
214 214 def _process_tree_items(self, items, cur_dir):
215 215 for item, stat_, id_, type_ in items:
216 216 if cur_dir:
217 217 name = '/'.join((cur_dir, item))
218 218 else:
219 219 name = item
220 220 self._paths[name] = [id_, type_]
221 221 self._stat_modes[name] = stat_
222 222
223 223 def _get_kind(self, path):
224 224 path_id, type_ = self._get_id_for_path(path)
225 225 if type_ == 'blob':
226 226 return NodeKind.FILE
227 227 elif type_ == 'tree':
228 228 return NodeKind.DIR
229 229 elif type == 'link':
230 230 return NodeKind.SUBMODULE
231 231 return None
232 232
233 233 def _get_filectx(self, path):
234 234 path = self._fix_path(path)
235 235 if self._get_kind(path) != NodeKind.FILE:
236 236 raise CommitError(
237 237 "File does not exist for commit %s at '%s'" %
238 238 (self.raw_id, path))
239 239 return path
240 240
241 241 def _get_file_nodes(self):
242 242 return chain(*(t[2] for t in self.walk()))
243 243
244 244 @LazyProperty
245 245 def parents(self):
246 246 """
247 247 Returns list of parent commits.
248 248 """
249 249 parent_ids = self._remote.commit_attribute(
250 250 self.id, self._parents_property)
251 251 return self._make_commits(parent_ids)
252 252
253 253 @LazyProperty
254 254 def children(self):
255 255 """
256 256 Returns list of child commits.
257 257 """
258 258 rev_filter = settings.GIT_REV_FILTER
259 259 output, __ = self.repository.run_git_command(
260 260 ['rev-list', '--children'] + rev_filter)
261 261
262 262 child_ids = []
263 263 pat = re.compile(r'^%s' % self.raw_id)
264 264 for l in output.splitlines():
265 265 if pat.match(l):
266 266 found_ids = l.split(' ')[1:]
267 267 child_ids.extend(found_ids)
268 268 return self._make_commits(child_ids)
269 269
270 270 def _make_commits(self, commit_ids, pre_load=None):
271 271 return [
272 272 self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
273 273 for commit_id in commit_ids]
274 274
275 275 def get_file_mode(self, path):
276 276 """
277 277 Returns stat mode of the file at the given `path`.
278 278 """
279 279 path = safe_str(path)
280 280 # ensure path is traversed
281 281 self._get_id_for_path(path)
282 282 return self._stat_modes[path]
283 283
284 284 def is_link(self, path):
285 285 return stat.S_ISLNK(self.get_file_mode(path))
286 286
287 287 def get_file_content(self, path):
288 288 """
289 289 Returns content of the file at given `path`.
290 290 """
291 291 id_, _ = self._get_id_for_path(path)
292 292 return self._remote.blob_as_pretty_string(id_)
293 293
294 294 def get_file_size(self, path):
295 295 """
296 296 Returns size of the file at given `path`.
297 297 """
298 298 id_, _ = self._get_id_for_path(path)
299 299 return self._remote.blob_raw_length(id_)
300 300
301 301 def get_file_history(self, path, limit=None, pre_load=None):
302 302 """
303 303 Returns history of file as reversed list of `GitCommit` objects for
304 304 which file at given `path` has been modified.
305 305
306 306 TODO: This function now uses an underlying 'git' command which works
307 307 quickly but ideally we should replace with an algorithm.
308 308 """
309 309 self._get_filectx(path)
310 310 f_path = safe_str(path)
311 311
312 312 cmd = ['log']
313 313 if limit:
314 314 cmd.extend(['-n', str(safe_int(limit, 0))])
315 315 cmd.extend(['--pretty=format: %H', '-s', self.raw_id, '--', f_path])
316 316
317 317 output, __ = self.repository.run_git_command(cmd)
318 318 commit_ids = re.findall(r'[0-9a-fA-F]{40}', output)
319 319
320 320 return [
321 321 self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
322 322 for commit_id in commit_ids]
323 323
324 324 # TODO: unused for now potential replacement for subprocess
325 325 def get_file_history_2(self, path, limit=None, pre_load=None):
326 326 """
327 327 Returns history of file as reversed list of `Commit` objects for
328 328 which file at given `path` has been modified.
329 329 """
330 330 self._get_filectx(path)
331 331 f_path = safe_str(path)
332 332
333 333 commit_ids = self._remote.get_file_history(f_path, self.id, limit)
334 334
335 335 return [
336 336 self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
337 337 for commit_id in commit_ids]
338 338
339 339 def get_file_annotate(self, path, pre_load=None):
340 340 """
341 341 Returns a generator of four element tuples with
342 342 lineno, commit_id, commit lazy loader and line
343 343
344 344 TODO: This function now uses os underlying 'git' command which is
345 345 generally not good. Should be replaced with algorithm iterating
346 346 commits.
347 347 """
348 348 cmd = ['blame', '-l', '--root', '-r', self.raw_id, '--', path]
349 349 # -l ==> outputs long shas (and we need all 40 characters)
350 350 # --root ==> doesn't put '^' character for bounderies
351 351 # -r commit_id ==> blames for the given commit
352 352 output, __ = self.repository.run_git_command(cmd)
353 353
354 354 for i, blame_line in enumerate(output.split('\n')[:-1]):
355 355 line_no = i + 1
356 356 commit_id, line = re.split(r' ', blame_line, 1)
357 357 yield (
358 358 line_no, commit_id,
359 359 lambda: self.repository.get_commit(commit_id=commit_id,
360 360 pre_load=pre_load),
361 361 line)
362 362
363 363 def get_nodes(self, path):
364 364 if self._get_kind(path) != NodeKind.DIR:
365 365 raise CommitError(
366 366 "Directory does not exist for commit %s at "
367 367 " '%s'" % (self.raw_id, path))
368 368 path = self._fix_path(path)
369 369 id_, _ = self._get_id_for_path(path)
370 370 tree_id = self._remote[id_]['id']
371 371 dirnodes = []
372 372 filenodes = []
373 373 alias = self.repository.alias
374 374 for name, stat_, id_, type_ in self._remote.tree_items(tree_id):
375 375 if type_ == 'link':
376 376 url = self._get_submodule_url('/'.join((path, name)))
377 377 dirnodes.append(SubModuleNode(
378 378 name, url=url, commit=id_, alias=alias))
379 379 continue
380 380
381 381 if path != '':
382 382 obj_path = '/'.join((path, name))
383 383 else:
384 384 obj_path = name
385 385 if obj_path not in self._stat_modes:
386 386 self._stat_modes[obj_path] = stat_
387 387
388 388 if type_ == 'tree':
389 389 dirnodes.append(DirNode(obj_path, commit=self))
390 390 elif type_ == 'blob':
391 391 filenodes.append(FileNode(obj_path, commit=self, mode=stat_))
392 392 else:
393 393 raise CommitError(
394 394 "Requested object should be Tree or Blob, is %s", type_)
395 395
396 396 nodes = dirnodes + filenodes
397 397 for node in nodes:
398 398 if node.path not in self.nodes:
399 399 self.nodes[node.path] = node
400 400 nodes.sort()
401 401 return nodes
402 402
403 403 def get_node(self, path, pre_load=None):
404 404 if isinstance(path, unicode):
405 405 path = path.encode('utf-8')
406 406 path = self._fix_path(path)
407 407 if path not in self.nodes:
408 408 try:
409 409 id_, type_ = self._get_id_for_path(path)
410 410 except CommitError:
411 411 raise NodeDoesNotExistError(
412 412 "Cannot find one of parents' directories for a given "
413 413 "path: %s" % path)
414 414
415 415 if type_ == 'link':
416 416 url = self._get_submodule_url(path)
417 417 node = SubModuleNode(path, url=url, commit=id_,
418 418 alias=self.repository.alias)
419 419 elif type_ == 'tree':
420 420 if path == '':
421 421 node = RootNode(commit=self)
422 422 else:
423 423 node = DirNode(path, commit=self)
424 424 elif type_ == 'blob':
425 425 node = FileNode(path, commit=self, pre_load=pre_load)
426 426 else:
427 427 raise self.no_node_at_path(path)
428 428
429 429 # cache node
430 430 self.nodes[path] = node
431 431 return self.nodes[path]
432 432
433 433 def get_largefile_node(self, path):
434 434 id_, _ = self._get_id_for_path(path)
435 435 pointer_spec = self._remote.is_large_file(id_)
436 436
437 437 if pointer_spec:
438 438 # content of that file regular FileNode is the hash of largefile
439 439 file_id = pointer_spec.get('oid_hash')
440 440 if self._remote.in_largefiles_store(file_id):
441 441 lf_path = self._remote.store_path(file_id)
442 442 return LargeFileNode(lf_path, commit=self, org_path=path)
443 443
444 444 @LazyProperty
445 445 def affected_files(self):
446 446 """
447 447 Gets a fast accessible file changes for given commit
448 448 """
449 449 added, modified, deleted = self._changes_cache
450 450 return list(added.union(modified).union(deleted))
451 451
452 452 @LazyProperty
453 453 def _changes_cache(self):
454 454 added = set()
455 455 modified = set()
456 456 deleted = set()
457 457 _r = self._remote
458 458
459 459 parents = self.parents
460 460 if not self.parents:
461 461 parents = [base.EmptyCommit()]
462 462 for parent in parents:
463 463 if isinstance(parent, base.EmptyCommit):
464 464 oid = None
465 465 else:
466 466 oid = parent.raw_id
467 467 changes = _r.tree_changes(oid, self.raw_id)
468 468 for (oldpath, newpath), (_, _), (_, _) in changes:
469 469 if newpath and oldpath:
470 470 modified.add(newpath)
471 471 elif newpath and not oldpath:
472 472 added.add(newpath)
473 473 elif not newpath and oldpath:
474 474 deleted.add(oldpath)
475 475 return added, modified, deleted
476 476
477 477 def _get_paths_for_status(self, status):
478 478 """
479 479 Returns sorted list of paths for given ``status``.
480 480
481 481 :param status: one of: *added*, *modified* or *deleted*
482 482 """
483 483 added, modified, deleted = self._changes_cache
484 484 return sorted({
485 485 'added': list(added),
486 486 'modified': list(modified),
487 487 'deleted': list(deleted)}[status]
488 488 )
489 489
490 490 @LazyProperty
491 491 def added(self):
492 492 """
493 493 Returns list of added ``FileNode`` objects.
494 494 """
495 495 if not self.parents:
496 496 return list(self._get_file_nodes())
497 497 return AddedFileNodesGenerator(
498 498 [n for n in self._get_paths_for_status('added')], self)
499 499
500 500 @LazyProperty
501 501 def changed(self):
502 502 """
503 503 Returns list of modified ``FileNode`` objects.
504 504 """
505 505 if not self.parents:
506 506 return []
507 507 return ChangedFileNodesGenerator(
508 508 [n for n in self._get_paths_for_status('modified')], self)
509 509
510 510 @LazyProperty
511 511 def removed(self):
512 512 """
513 513 Returns list of removed ``FileNode`` objects.
514 514 """
515 515 if not self.parents:
516 516 return []
517 517 return RemovedFileNodesGenerator(
518 518 [n for n in self._get_paths_for_status('deleted')], self)
519 519
520 520 def _get_submodule_url(self, submodule_path):
521 521 git_modules_path = '.gitmodules'
522 522
523 523 if self._submodules is None:
524 524 self._submodules = {}
525 525
526 526 try:
527 527 submodules_node = self.get_node(git_modules_path)
528 528 except NodeDoesNotExistError:
529 529 return None
530 530
531 531 content = submodules_node.content
532 532
533 533 # ConfigParser fails if there are whitespaces
534 534 content = '\n'.join(l.strip() for l in content.split('\n'))
535 535
536 parser = ConfigParser()
536 parser = configparser.ConfigParser()
537 537 parser.readfp(StringIO(content))
538 538
539 539 for section in parser.sections():
540 540 path = parser.get(section, 'path')
541 541 url = parser.get(section, 'url')
542 542 if path and url:
543 543 self._submodules[path.strip('/')] = url
544 544
545 545 return self._submodules.get(submodule_path.strip('/'))
General Comments 0
You need to be logged in to leave comments. Login now