##// END OF EJS Templates
system-info: fetch vcs settings from vcsserver. Fixes #4276...
marcink -
r1111:5dd5a063 default
parent child Browse files
Show More
This diff has been collapsed as it changes many lines, (582 lines changed) Show them Hide them
@@ -0,0 +1,582 b''
1 import os
2 import sys
3 import time
4 import platform
5 import pkg_resources
6 import logging
7
8
9 log = logging.getLogger(__name__)
10
11
12 psutil = None
13
14 try:
15 # cygwin cannot have yet psutil support.
16 import psutil as psutil
17 except ImportError:
18 pass
19
20
21 _NA = 'NOT AVAILABLE'
22
23 STATE_OK = 'ok'
24 STATE_ERR = 'error'
25 STATE_WARN = 'warning'
26
27 STATE_OK_DEFAULT = {'message': '', 'type': STATE_OK}
28
29
30 # HELPERS
31 def percentage(part, whole):
32 whole = float(whole)
33 if whole > 0:
34 return 100 * float(part) / whole
35 return 0
36
37
38 def get_storage_size(storage_path):
39 sizes = []
40 for file_ in os.listdir(storage_path):
41 storage_file = os.path.join(storage_path, file_)
42 if os.path.isfile(storage_file):
43 try:
44 sizes.append(os.path.getsize(storage_file))
45 except OSError:
46 log.exception('Failed to get size of storage file %s',
47 storage_file)
48 pass
49
50 return sum(sizes)
51
52
53 class SysInfoRes(object):
54 def __init__(self, value, state=STATE_OK_DEFAULT, human_value=None):
55 self.value = value
56 self.state = state
57 self.human_value = human_value or value
58
59 def __json__(self):
60 return {
61 'value': self.value,
62 'state': self.state,
63 'human_value': self.human_value,
64 }
65
66 def __str__(self):
67 return '<SysInfoRes({})>'.format(self.__json__())
68
69
70 class SysInfo(object):
71
72 def __init__(self, func_name, **kwargs):
73 self.func_name = func_name
74 self.value = _NA
75 self.state = None
76 self.kwargs = kwargs or {}
77
78 def __call__(self):
79 computed = self.compute(**self.kwargs)
80 if not isinstance(computed, SysInfoRes):
81 raise ValueError(
82 'computed value for {} is not instance of '
83 '{}, got {} instead'.format(
84 self.func_name, SysInfoRes, type(computed)))
85 return computed.__json__()
86
87 def __str__(self):
88 return '<SysInfo({})>'.format(self.func_name)
89
90 def compute(self, **kwargs):
91 return self.func_name(**kwargs)
92
93
94 # SysInfo functions
95 def python_info():
96 value = {
97 'version': ' '.join(platform._sys_version()),
98 'executable': sys.executable
99 }
100 return SysInfoRes(value=value)
101
102
103 def py_modules():
104 mods = dict([(p.project_name, p.version)
105 for p in pkg_resources.working_set])
106 value = sorted(mods.items(), key=lambda k: k[0].lower())
107 return SysInfoRes(value=value)
108
109
110 def platform_type():
111 from rhodecode.lib.utils import safe_unicode
112 value = safe_unicode(platform.platform())
113 return SysInfoRes(value=value)
114
115
116 def uptime():
117 from rhodecode.lib.helpers import age, time_to_datetime
118 from rhodecode.translation import _
119
120 _uptime = _uptime_human = {'boot_time': 0, 'uptime': 0}
121 state = STATE_OK_DEFAULT
122 if not psutil:
123 return SysInfoRes(value=_uptime, state=state)
124
125 boot_time = psutil.boot_time()
126 _uptime['boot_time'] = boot_time
127 _uptime['uptime'] = time.time() - boot_time
128
129 _uptime_human['boot_time'] = time_to_datetime(boot_time)
130 _uptime_human['uptime'] = _('Server started {}').format(
131 age(time_to_datetime(boot_time)))
132
133 return SysInfoRes(value=_uptime, human_value=_uptime_human)
134
135
136 def memory():
137 from rhodecode.lib.helpers import format_byte_size_binary
138 _memory = {'available': 0, 'used': 0, 'cached': 0, 'percent': 0,
139 'percent_used': 0, 'free': 0, 'inactive': 0, 'active': 0,
140 'shared': 0, 'total': 0, 'buffers': 0}
141 state = STATE_OK_DEFAULT
142 if not psutil:
143 return SysInfoRes(value=_memory, state=state)
144
145 # memory
146 _memory = dict(psutil.virtual_memory()._asdict())
147 _memory['percent_used'] = psutil._common.usage_percent(
148 (_memory['total'] - _memory['free']), _memory['total'], 1)
149
150 try:
151 human_value = '%s/%s, %s%% used' % (
152 format_byte_size_binary(_memory['used']),
153 format_byte_size_binary(_memory['total']),
154 _memory['percent_used'],)
155 except TypeError:
156 human_value = 'NOT AVAILABLE'
157
158 if state['type'] == STATE_OK and _memory['percent_used'] > 90:
159 msg = 'Critical: your available RAM memory is very low.'
160 state = {'message': msg, 'type': STATE_ERR}
161
162 elif state['type'] == STATE_OK and _memory['percent_used'] > 70:
163 msg = 'Warning: your available RAM memory is running low.'
164 state = {'message': msg, 'type': STATE_WARN}
165
166 return SysInfoRes(value=_memory, state=state, human_value=human_value)
167
168
169 def machine_load():
170 _load = {'1_min': _NA, '5_min': _NA, '15_min': _NA}
171
172 state = STATE_OK_DEFAULT
173 if not psutil:
174 return SysInfoRes(value=_load, state=state)
175
176 # load averages
177 if hasattr(psutil.os, 'getloadavg'):
178 _load = dict(zip(['1_min', '5_min', '15_min'], psutil.os.getloadavg()))
179
180 human_value = '1min: %s, 5min: %s, 15min: %s' % (
181 _load['1_min'], _load['5_min'], _load['15_min'])
182
183 # TODO: warn about too-much load 15 min
184 return SysInfoRes(value=_load, state=state, human_value=human_value)
185
186
187 def cpu():
188 state = STATE_OK_DEFAULT
189 cpu_value = 0
190 if not psutil:
191 return SysInfoRes(value=cpu_value, state=state)
192
193 cpu_value = psutil.cpu_percent(0.5)
194 human_value = '{} %'.format(cpu_value)
195 return SysInfoRes(value=cpu_value, state=state, human_value=human_value)
196
197
198 def storage():
199 from rhodecode.lib.helpers import format_byte_size_binary
200 from rhodecode.model.settings import VcsSettingsModel
201 path = VcsSettingsModel().get_repos_location()
202
203 # disk storage
204 disk = {'percent': 0, 'used': 0, 'total': 0, 'path': path, 'text': ''}
205 state = STATE_OK_DEFAULT
206 if not psutil:
207 return SysInfoRes(value=disk, state=state)
208
209 try:
210 disk.update(dict(psutil.disk_usage(path)._asdict()))
211 except Exception as e:
212 log.exception('Failed to fetch disk info')
213 state = {'message': str(e), 'type': STATE_ERR}
214
215 human_value = disk
216 human_value['text'] = "{}/{}, {}% used".format(
217 format_byte_size_binary(disk['used']),
218 format_byte_size_binary(disk['total']),
219 (disk['percent']))
220
221 if state['type'] == STATE_OK and disk['percent'] > 90:
222 msg = 'Critical: your disk space is very low.'
223 state = {'message': msg, 'type': STATE_ERR}
224
225 elif state['type'] == STATE_OK and disk['percent'] > 70:
226 msg = 'Warning: your disk space is running low.'
227 state = {'message': msg, 'type': STATE_WARN}
228
229 return SysInfoRes(value=disk, state=state, human_value=human_value)
230
231
232 def storage_inodes():
233 from rhodecode.model.settings import VcsSettingsModel
234 path = VcsSettingsModel().get_repos_location()
235
236 _disk_inodes = dict(percent=0, free=0, used=0, total=0, path=path, text='')
237 state = STATE_OK_DEFAULT
238 if not psutil:
239 return SysInfoRes(value=_disk_inodes, state=state)
240
241 try:
242 i_stat = os.statvfs(path)
243
244 _disk_inodes['used'] = i_stat.f_ffree
245 _disk_inodes['free'] = i_stat.f_favail
246 _disk_inodes['total'] = i_stat.f_files
247 _disk_inodes['percent'] = percentage(
248 _disk_inodes['used'], _disk_inodes['total'])
249 except Exception as e:
250 log.exception('Failed to fetch disk inodes info')
251 state = {'message': str(e), 'type': STATE_ERR}
252
253 human_value = _disk_inodes
254 human_value['text'] = "{}/{}, {}% used".format(
255 _disk_inodes['used'], _disk_inodes['total'], _disk_inodes['percent'])
256
257 if state['type'] == STATE_OK and _disk_inodes['percent'] > 90:
258 msg = 'Critical: your disk free inodes are very low.'
259 state = {'message': msg, 'type': STATE_ERR}
260
261 elif state['type'] == STATE_OK and _disk_inodes['percent'] > 70:
262 msg = 'Warning: your disk free inodes are running low.'
263 state = {'message': msg, 'type': STATE_WARN}
264
265 return SysInfoRes(value=_disk_inodes, state=state)
266
267
268 def storage_archives():
269 import rhodecode
270 from rhodecode.lib.utils import safe_str
271 from rhodecode.lib.helpers import format_byte_size_binary
272
273 msg = 'Enable this by setting ' \
274 'archive_cache_dir=/path/to/cache option in the .ini file'
275 path = safe_str(rhodecode.CONFIG.get('archive_cache_dir', msg))
276
277 disk_archive = dict(percent=0, used=0, total=0, items=0, path=path, text='')
278 state = STATE_OK_DEFAULT
279 try:
280 items_count = 0
281 used = 0
282 for root, dirs, files in os.walk(path):
283 if root == path:
284 items_count = len(dirs)
285
286 for f in files:
287 try:
288 used += os.path.getsize(os.path.join(root, f))
289 except OSError:
290 pass
291 disk_archive.update({
292 'percent': 100,
293 'used': used,
294 'total': used,
295 'items': items_count
296 })
297
298 except Exception as e:
299 log.exception('failed to fetch archive cache storage')
300 state = {'message': str(e), 'type': STATE_ERR}
301
302 human_value = disk_archive
303 human_value['text'] = "{} ({} items)".format(format_byte_size_binary(
304 disk_archive['used']), disk_archive['total'])
305
306 return SysInfoRes(value=disk_archive, state=state, human_value=human_value)
307
308
309 def storage_gist():
310 from rhodecode.model.gist import GIST_STORE_LOC
311 from rhodecode.model.settings import VcsSettingsModel
312 from rhodecode.lib.utils import safe_str
313 from rhodecode.lib.helpers import format_byte_size_binary
314 path = safe_str(os.path.join(
315 VcsSettingsModel().get_repos_location(), GIST_STORE_LOC))
316
317 # gist storage
318 _disk_gist = dict(percent=0, used=0, total=0, items=0, path=path, text='')
319 state = STATE_OK_DEFAULT
320
321 try:
322 items_count = 0
323 used = 0
324 for root, dirs, files in os.walk(path):
325 if root == path:
326 items_count = len(dirs)
327
328 for f in files:
329 try:
330 used += os.path.getsize(os.path.join(root, f))
331 except OSError:
332 pass
333 _disk_gist.update({
334 'percent': 100,
335 'used': used,
336 'total': used,
337 'items': items_count
338 })
339 except Exception as e:
340 log.exception('failed to fetch gist storage items')
341 state = {'message': str(e), 'type': STATE_ERR}
342
343 human_value = _disk_gist
344 human_value['text'] = "{} ({} items)".format(format_byte_size_binary(
345 _disk_gist['used']), _disk_gist['items'])
346 return SysInfoRes(value=_disk_gist, state=state, human_value=human_value)
347
348
349 def storage_search():
350 import rhodecode
351 path = rhodecode.CONFIG.get('search.location', '')
352
353 # search index storage
354 _disk_index = dict(percent=0, used=0, total=0, path=path, text='')
355 state = STATE_OK_DEFAULT
356 try:
357 search_index_storage_path_exists = os.path.isdir(path)
358 if search_index_storage_path_exists:
359 used = get_storage_size(path)
360 _disk_index.update({
361 'percent': 100,
362 'used': used,
363 'total': used,
364 })
365 except Exception as e:
366 log.exception('failed to fetch search index storage')
367 state = {'message': str(e), 'type': STATE_ERR}
368
369 human_value = _disk_index
370 human_value['text'] = "{}/{}, {}% used".format(
371 _disk_index['used'], _disk_index['total'], _disk_index['percent'])
372
373 return SysInfoRes(value=_disk_index, state=state, human_value=human_value)
374
375
376 def git_info():
377 from rhodecode.lib.vcs.backends import git
378 state = STATE_OK_DEFAULT
379 value = human_value = ''
380 try:
381 value = git.discover_git_version(raise_on_exc=True)
382 human_value = 'version reported from VCSServer: {}'.format(value)
383 except Exception as e:
384 state = {'message': str(e), 'type': STATE_ERR}
385
386 return SysInfoRes(value=value, state=state, human_value=human_value)
387
388
389 def hg_info():
390 from rhodecode.lib.vcs.backends import hg
391 state = STATE_OK_DEFAULT
392 value = human_value = ''
393 try:
394 value = hg.discover_hg_version(raise_on_exc=True)
395 human_value = 'version reported from VCSServer: {}'.format(value)
396 except Exception as e:
397 state = {'message': str(e), 'type': STATE_ERR}
398 return SysInfoRes(value=value, state=state, human_value=human_value)
399
400
401 def svn_info():
402 from rhodecode.lib.vcs.backends import svn
403 state = STATE_OK_DEFAULT
404 value = human_value = ''
405 try:
406 value = svn.discover_svn_version(raise_on_exc=True)
407 human_value = 'version reported from VCSServer: {}'.format(value)
408 except Exception as e:
409 state = {'message': str(e), 'type': STATE_ERR}
410 return SysInfoRes(value=value, state=state, human_value=human_value)
411
412
413 def vcs_backends():
414 import rhodecode
415 value = rhodecode.CONFIG.get('vcs.backends', '').split(',')
416 human_value = 'Enabled backends in order: {}'.format(','.join(value))
417 return SysInfoRes(value=value, human_value=human_value)
418
419
420 def vcs_server():
421 import rhodecode
422 from rhodecode.lib.vcs.backends import get_vcsserver_version
423
424 server_url = rhodecode.CONFIG.get('vcs.server')
425 enabled = rhodecode.CONFIG.get('vcs.server.enable')
426 protocol = rhodecode.CONFIG.get('vcs.server.protocol')
427 state = STATE_OK_DEFAULT
428 version = None
429
430 try:
431 version = get_vcsserver_version()
432 connection = 'connected'
433 except Exception as e:
434 connection = 'failed'
435 state = {'message': str(e), 'type': STATE_ERR}
436
437 value = dict(
438 url=server_url,
439 enabled=enabled,
440 protocol=protocol,
441 connection=connection,
442 version=version,
443 text='',
444 )
445
446 human_value = value
447 human_value['text'] = \
448 '{url}@ver:{ver} via {mode} mode, connection:{conn}'.format(
449 url=server_url, ver=version, mode=protocol, conn=connection)
450
451 return SysInfoRes(value='', state=state, human_value=human_value)
452
453
454 def rhodecode_app_info():
455 import rhodecode
456 return SysInfoRes(value={'rhodecode_version': rhodecode.__version__})
457
458
459 def rhodecode_config():
460 import rhodecode
461 path = rhodecode.CONFIG.get('__file__')
462 rhodecode_ini_safe = rhodecode.CONFIG.copy()
463
464 blacklist = [
465 'rhodecode_license_key',
466 'routes.map',
467 'pylons.h',
468 'pylons.app_globals',
469 'pylons.environ_config',
470 'sqlalchemy.db1.url',
471 'channelstream.secret',
472 'beaker.session.secret',
473 'rhodecode.encrypted_values.secret',
474 'rhodecode_auth_github_consumer_key',
475 'rhodecode_auth_github_consumer_secret',
476 'rhodecode_auth_google_consumer_key',
477 'rhodecode_auth_google_consumer_secret',
478 'rhodecode_auth_bitbucket_consumer_secret',
479 'rhodecode_auth_bitbucket_consumer_key',
480 'rhodecode_auth_twitter_consumer_secret',
481 'rhodecode_auth_twitter_consumer_key',
482
483 'rhodecode_auth_twitter_secret',
484 'rhodecode_auth_github_secret',
485 'rhodecode_auth_google_secret',
486 'rhodecode_auth_bitbucket_secret',
487
488 'appenlight.api_key',
489 ('app_conf', 'sqlalchemy.db1.url')
490 ]
491 for k in blacklist:
492 if isinstance(k, tuple):
493 section, key = k
494 if section in rhodecode_ini_safe:
495 rhodecode_ini_safe[section] = '**OBFUSCATED**'
496 else:
497 rhodecode_ini_safe.pop(k, None)
498
499 # TODO: maybe put some CONFIG checks here ?
500 return SysInfoRes(value={'config': rhodecode_ini_safe, 'path': path})
501
502
503 def database_info():
504 import rhodecode
505 from sqlalchemy.engine import url as engine_url
506 from rhodecode.model.meta import Base as sql_base, Session
507 from rhodecode.model.db import DbMigrateVersion
508
509 state = STATE_OK_DEFAULT
510
511 db_migrate = DbMigrateVersion.query().filter(
512 DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one()
513
514 db_url_obj = engine_url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url'])
515
516 try:
517 engine = sql_base.metadata.bind
518 db_server_info = engine.dialect._get_server_version_info(
519 Session.connection(bind=engine))
520 db_version = '.'.join(map(str, db_server_info))
521 except Exception:
522 log.exception('failed to fetch db version')
523 db_version = 'UNKNOWN'
524
525 db_info = dict(
526 migrate_version=db_migrate.version,
527 type=db_url_obj.get_backend_name(),
528 version=db_version,
529 url=repr(db_url_obj)
530 )
531
532 human_value = db_info
533 human_value['url'] = "{} @ migration version: {}".format(
534 db_info['url'], db_info['migrate_version'])
535 human_value['version'] = "{} {}".format(db_info['type'], db_info['version'])
536 return SysInfoRes(value=db_info, state=state, human_value=human_value)
537
538
539 def server_info(environ):
540 import rhodecode
541 from rhodecode.lib.base import get_server_ip_addr, get_server_port
542
543 value = {
544 'server_ip': '%s:%s' % (
545 get_server_ip_addr(environ, log_errors=False),
546 get_server_port(environ)
547 ),
548 'server_id': rhodecode.CONFIG.get('instance_id'),
549 }
550 return SysInfoRes(value=value)
551
552
553 def get_system_info(environ):
554 environ = environ or {}
555 return {
556 'rhodecode_app': SysInfo(rhodecode_app_info)(),
557 'rhodecode_config': SysInfo(rhodecode_config)(),
558 'python': SysInfo(python_info)(),
559 'py_modules': SysInfo(py_modules)(),
560
561 'platform': SysInfo(platform_type)(),
562 'server': SysInfo(server_info, environ=environ)(),
563 'database': SysInfo(database_info)(),
564
565 'storage': SysInfo(storage)(),
566 'storage_inodes': SysInfo(storage_inodes)(),
567 'storage_archive': SysInfo(storage_archives)(),
568 'storage_search': SysInfo(storage_search)(),
569 'storage_gist': SysInfo(storage_gist)(),
570
571 'uptime': SysInfo(uptime)(),
572 'load': SysInfo(machine_load)(),
573 'cpu': SysInfo(cpu)(),
574 'memory': SysInfo(memory)(),
575
576 'vcs_backends': SysInfo(vcs_backends)(),
577 'vcs_server': SysInfo(vcs_server)(),
578
579 'git': SysInfo(git_info)(),
580 'hg': SysInfo(hg_info)(),
581 'svn': SysInfo(svn_info)(),
582 }
@@ -0,0 +1,15 b''
1 import py.test
2
3 from rhodecode.lib.system_info import get_system_info
4
5
6 def test_system_info(app):
7 info = get_system_info({})
8 assert info['load']['value']['15_min'] != 'NOT AVAILABLE'
9
10
11 def test_system_info_without_psutil(monkeypatch, app):
12 import rhodecode.lib.system_info
13 monkeypatch.setattr(rhodecode.lib.system_info, 'psutil', None)
14 info = get_system_info({})
15 assert info['load']['value']['15_min'] == 'NOT AVAILABLE'
@@ -1,67 +1,67 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21
22 22 import pytest
23 23
24 24 from rhodecode.model.scm import ScmModel
25 25 from rhodecode.api.tests.utils import build_data, api_call, assert_ok
26 26
27 27
28 28 @pytest.fixture
29 29 def http_host_stub():
30 30 """
31 31 To ensure that we can get an IP address, this test shall run with a
32 32 hostname set to "localhost".
33 33 """
34 34 return 'localhost:80'
35 35
36 36
37 37 @pytest.mark.usefixtures("testuser_api", "app")
38 38 class TestGetServerInfo(object):
39 39 def test_api_get_server_info(self):
40 40 id_, params = build_data(self.apikey, 'get_server_info')
41 41 response = api_call(self.app, params)
42 42 resp = response.json
43 43 expected = ScmModel().get_server_info()
44 44 expected['memory'] = resp['result']['memory']
45 45 expected['uptime'] = resp['result']['uptime']
46 46 expected['load'] = resp['result']['load']
47 47 expected['cpu'] = resp['result']['cpu']
48 expected['disk'] = resp['result']['disk']
49 expected['disk_inodes'] = resp['result']['disk_inodes']
50 expected['server_ip'] = '127.0.0.1:80'
48 expected['storage'] = resp['result']['storage']
49 expected['storage_inodes'] = resp['result']['storage_inodes']
50 expected['server'] = resp['result']['server']
51 51
52 52 assert_ok(id_, expected, given=response.body)
53 53
54 54 def test_api_get_server_info_ip(self):
55 55 id_, params = build_data(self.apikey, 'get_server_info')
56 56 response = api_call(self.app, params)
57 57 resp = response.json
58 58 expected = ScmModel().get_server_info({'SERVER_NAME': 'unknown'})
59 59 expected['memory'] = resp['result']['memory']
60 60 expected['uptime'] = resp['result']['uptime']
61 61 expected['load'] = resp['result']['load']
62 62 expected['cpu'] = resp['result']['cpu']
63 expected['disk'] = resp['result']['disk']
64 expected['disk_inodes'] = resp['result']['disk_inodes']
65 expected['server_ip'] = '127.0.0.1:80'
63 expected['storage'] = resp['result']['storage']
64 expected['storage_inodes'] = resp['result']['storage_inodes']
65 expected['server'] = resp['result']['server']
66 66
67 67 assert_ok(id_, expected, given=response.body)
@@ -1,814 +1,837 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21
22 22 """
23 23 settings controller for rhodecode admin
24 24 """
25 25
26 26 import collections
27 27 import logging
28 28 import urllib2
29 29
30 30 import datetime
31 31 import formencode
32 32 from formencode import htmlfill
33 33 import packaging.version
34 34 from pylons import request, tmpl_context as c, url, config
35 35 from pylons.controllers.util import redirect
36 36 from pylons.i18n.translation import _, lazy_ugettext
37 37 from pyramid.threadlocal import get_current_registry
38 38 from webob.exc import HTTPBadRequest
39 39
40 40 import rhodecode
41 41 from rhodecode.admin.navigation import navigation_list
42 42 from rhodecode.lib import auth
43 43 from rhodecode.lib import helpers as h
44 44 from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
45 45 from rhodecode.lib.base import BaseController, render
46 46 from rhodecode.lib.celerylib import tasks, run_task
47 47 from rhodecode.lib.utils import repo2db_mapper
48 48 from rhodecode.lib.utils2 import (
49 49 str2bool, safe_unicode, AttributeDict, safe_int)
50 50 from rhodecode.lib.compat import OrderedDict
51 51 from rhodecode.lib.ext_json import json
52 52 from rhodecode.lib.utils import jsonify
53 53
54 54 from rhodecode.model.db import RhodeCodeUi, Repository
55 55 from rhodecode.model.forms import ApplicationSettingsForm, \
56 56 ApplicationUiSettingsForm, ApplicationVisualisationForm, \
57 57 LabsSettingsForm, IssueTrackerPatternsForm
58 58 from rhodecode.model.repo_group import RepoGroupModel
59 59
60 60 from rhodecode.model.scm import ScmModel
61 61 from rhodecode.model.notification import EmailNotificationModel
62 62 from rhodecode.model.meta import Session
63 63 from rhodecode.model.settings import (
64 64 IssueTrackerSettingsModel, VcsSettingsModel, SettingNotFound,
65 65 SettingsModel)
66 66
67 67 from rhodecode.model.supervisor import SupervisorModel, SUPERVISOR_MASTER
68 68 from rhodecode.svn_support.config_keys import generate_config
69 69
70 70
71 71 log = logging.getLogger(__name__)
72 72
73 73
74 74 class SettingsController(BaseController):
75 75 """REST Controller styled on the Atom Publishing Protocol"""
76 76 # To properly map this controller, ensure your config/routing.py
77 77 # file has a resource setup:
78 78 # map.resource('setting', 'settings', controller='admin/settings',
79 79 # path_prefix='/admin', name_prefix='admin_')
80 80
81 81 @LoginRequired()
82 82 def __before__(self):
83 83 super(SettingsController, self).__before__()
84 84 c.labs_active = str2bool(
85 85 rhodecode.CONFIG.get('labs_settings_active', 'true'))
86 86 c.navlist = navigation_list(request)
87 87
88 88 def _get_hg_ui_settings(self):
89 89 ret = RhodeCodeUi.query().all()
90 90
91 91 if not ret:
92 92 raise Exception('Could not get application ui settings !')
93 93 settings = {}
94 94 for each in ret:
95 95 k = each.ui_key
96 96 v = each.ui_value
97 97 if k == '/':
98 98 k = 'root_path'
99 99
100 100 if k in ['push_ssl', 'publish']:
101 101 v = str2bool(v)
102 102
103 103 if k.find('.') != -1:
104 104 k = k.replace('.', '_')
105 105
106 106 if each.ui_section in ['hooks', 'extensions']:
107 107 v = each.ui_active
108 108
109 109 settings[each.ui_section + '_' + k] = v
110 110 return settings
111 111
112 112 @HasPermissionAllDecorator('hg.admin')
113 113 @auth.CSRFRequired()
114 114 @jsonify
115 115 def delete_svn_pattern(self):
116 116 if not request.is_xhr:
117 117 raise HTTPBadRequest()
118 118
119 119 delete_pattern_id = request.POST.get('delete_svn_pattern')
120 120 model = VcsSettingsModel()
121 121 try:
122 122 model.delete_global_svn_pattern(delete_pattern_id)
123 123 except SettingNotFound:
124 124 raise HTTPBadRequest()
125 125
126 126 Session().commit()
127 127 return True
128 128
129 129 @HasPermissionAllDecorator('hg.admin')
130 130 @auth.CSRFRequired()
131 131 def settings_vcs_update(self):
132 132 """POST /admin/settings: All items in the collection"""
133 133 # url('admin_settings_vcs')
134 134 c.active = 'vcs'
135 135
136 136 model = VcsSettingsModel()
137 137 c.svn_branch_patterns = model.get_global_svn_branch_patterns()
138 138 c.svn_tag_patterns = model.get_global_svn_tag_patterns()
139 139
140 140 # TODO: Replace with request.registry after migrating to pyramid.
141 141 pyramid_settings = get_current_registry().settings
142 142 c.svn_proxy_generate_config = pyramid_settings[generate_config]
143 143
144 144 application_form = ApplicationUiSettingsForm()()
145 145
146 146 try:
147 147 form_result = application_form.to_python(dict(request.POST))
148 148 except formencode.Invalid as errors:
149 149 h.flash(
150 150 _("Some form inputs contain invalid data."),
151 151 category='error')
152 152 return htmlfill.render(
153 153 render('admin/settings/settings.html'),
154 154 defaults=errors.value,
155 155 errors=errors.error_dict or {},
156 156 prefix_error=False,
157 157 encoding="UTF-8",
158 158 force_defaults=False
159 159 )
160 160
161 161 try:
162 162 if c.visual.allow_repo_location_change:
163 163 model.update_global_path_setting(
164 164 form_result['paths_root_path'])
165 165
166 166 model.update_global_ssl_setting(form_result['web_push_ssl'])
167 167 model.update_global_hook_settings(form_result)
168 168
169 169 model.create_or_update_global_svn_settings(form_result)
170 170 model.create_or_update_global_hg_settings(form_result)
171 171 model.create_or_update_global_pr_settings(form_result)
172 172 except Exception:
173 173 log.exception("Exception while updating settings")
174 174 h.flash(_('Error occurred during updating '
175 175 'application settings'), category='error')
176 176 else:
177 177 Session().commit()
178 178 h.flash(_('Updated VCS settings'), category='success')
179 179 return redirect(url('admin_settings_vcs'))
180 180
181 181 return htmlfill.render(
182 182 render('admin/settings/settings.html'),
183 183 defaults=self._form_defaults(),
184 184 encoding="UTF-8",
185 185 force_defaults=False)
186 186
187 187 @HasPermissionAllDecorator('hg.admin')
188 188 def settings_vcs(self):
189 189 """GET /admin/settings: All items in the collection"""
190 190 # url('admin_settings_vcs')
191 191 c.active = 'vcs'
192 192 model = VcsSettingsModel()
193 193 c.svn_branch_patterns = model.get_global_svn_branch_patterns()
194 194 c.svn_tag_patterns = model.get_global_svn_tag_patterns()
195 195
196 196 # TODO: Replace with request.registry after migrating to pyramid.
197 197 pyramid_settings = get_current_registry().settings
198 198 c.svn_proxy_generate_config = pyramid_settings[generate_config]
199 199
200 200 return htmlfill.render(
201 201 render('admin/settings/settings.html'),
202 202 defaults=self._form_defaults(),
203 203 encoding="UTF-8",
204 204 force_defaults=False)
205 205
206 206 @HasPermissionAllDecorator('hg.admin')
207 207 @auth.CSRFRequired()
208 208 def settings_mapping_update(self):
209 209 """POST /admin/settings/mapping: All items in the collection"""
210 210 # url('admin_settings_mapping')
211 211 c.active = 'mapping'
212 212 rm_obsolete = request.POST.get('destroy', False)
213 213 invalidate_cache = request.POST.get('invalidate', False)
214 214 log.debug(
215 215 'rescanning repo location with destroy obsolete=%s', rm_obsolete)
216 216
217 217 if invalidate_cache:
218 218 log.debug('invalidating all repositories cache')
219 219 for repo in Repository.get_all():
220 220 ScmModel().mark_for_invalidation(repo.repo_name, delete=True)
221 221
222 222 filesystem_repos = ScmModel().repo_scan()
223 223 added, removed = repo2db_mapper(filesystem_repos, rm_obsolete)
224 224 _repr = lambda l: ', '.join(map(safe_unicode, l)) or '-'
225 225 h.flash(_('Repositories successfully '
226 226 'rescanned added: %s ; removed: %s') %
227 227 (_repr(added), _repr(removed)),
228 228 category='success')
229 229 return redirect(url('admin_settings_mapping'))
230 230
231 231 @HasPermissionAllDecorator('hg.admin')
232 232 def settings_mapping(self):
233 233 """GET /admin/settings/mapping: All items in the collection"""
234 234 # url('admin_settings_mapping')
235 235 c.active = 'mapping'
236 236
237 237 return htmlfill.render(
238 238 render('admin/settings/settings.html'),
239 239 defaults=self._form_defaults(),
240 240 encoding="UTF-8",
241 241 force_defaults=False)
242 242
243 243 @HasPermissionAllDecorator('hg.admin')
244 244 @auth.CSRFRequired()
245 245 def settings_global_update(self):
246 246 """POST /admin/settings/global: All items in the collection"""
247 247 # url('admin_settings_global')
248 248 c.active = 'global'
249 249 c.personal_repo_group_default_pattern = RepoGroupModel()\
250 250 .get_personal_group_name_pattern()
251 251 application_form = ApplicationSettingsForm()()
252 252 try:
253 253 form_result = application_form.to_python(dict(request.POST))
254 254 except formencode.Invalid as errors:
255 255 return htmlfill.render(
256 256 render('admin/settings/settings.html'),
257 257 defaults=errors.value,
258 258 errors=errors.error_dict or {},
259 259 prefix_error=False,
260 260 encoding="UTF-8",
261 261 force_defaults=False)
262 262
263 263 try:
264 264 settings = [
265 265 ('title', 'rhodecode_title', 'unicode'),
266 266 ('realm', 'rhodecode_realm', 'unicode'),
267 267 ('pre_code', 'rhodecode_pre_code', 'unicode'),
268 268 ('post_code', 'rhodecode_post_code', 'unicode'),
269 269 ('captcha_public_key', 'rhodecode_captcha_public_key', 'unicode'),
270 270 ('captcha_private_key', 'rhodecode_captcha_private_key', 'unicode'),
271 271 ('create_personal_repo_group', 'rhodecode_create_personal_repo_group', 'bool'),
272 272 ('personal_repo_group_pattern', 'rhodecode_personal_repo_group_pattern', 'unicode'),
273 273 ]
274 274 for setting, form_key, type_ in settings:
275 275 sett = SettingsModel().create_or_update_setting(
276 276 setting, form_result[form_key], type_)
277 277 Session().add(sett)
278 278
279 279 Session().commit()
280 280 SettingsModel().invalidate_settings_cache()
281 281 h.flash(_('Updated application settings'), category='success')
282 282 except Exception:
283 283 log.exception("Exception while updating application settings")
284 284 h.flash(
285 285 _('Error occurred during updating application settings'),
286 286 category='error')
287 287
288 288 return redirect(url('admin_settings_global'))
289 289
290 290 @HasPermissionAllDecorator('hg.admin')
291 291 def settings_global(self):
292 292 """GET /admin/settings/global: All items in the collection"""
293 293 # url('admin_settings_global')
294 294 c.active = 'global'
295 295 c.personal_repo_group_default_pattern = RepoGroupModel()\
296 296 .get_personal_group_name_pattern()
297 297
298 298 return htmlfill.render(
299 299 render('admin/settings/settings.html'),
300 300 defaults=self._form_defaults(),
301 301 encoding="UTF-8",
302 302 force_defaults=False)
303 303
304 304 @HasPermissionAllDecorator('hg.admin')
305 305 @auth.CSRFRequired()
306 306 def settings_visual_update(self):
307 307 """POST /admin/settings/visual: All items in the collection"""
308 308 # url('admin_settings_visual')
309 309 c.active = 'visual'
310 310 application_form = ApplicationVisualisationForm()()
311 311 try:
312 312 form_result = application_form.to_python(dict(request.POST))
313 313 except formencode.Invalid as errors:
314 314 return htmlfill.render(
315 315 render('admin/settings/settings.html'),
316 316 defaults=errors.value,
317 317 errors=errors.error_dict or {},
318 318 prefix_error=False,
319 319 encoding="UTF-8",
320 320 force_defaults=False
321 321 )
322 322
323 323 try:
324 324 settings = [
325 325 ('show_public_icon', 'rhodecode_show_public_icon', 'bool'),
326 326 ('show_private_icon', 'rhodecode_show_private_icon', 'bool'),
327 327 ('stylify_metatags', 'rhodecode_stylify_metatags', 'bool'),
328 328 ('repository_fields', 'rhodecode_repository_fields', 'bool'),
329 329 ('dashboard_items', 'rhodecode_dashboard_items', 'int'),
330 330 ('admin_grid_items', 'rhodecode_admin_grid_items', 'int'),
331 331 ('show_version', 'rhodecode_show_version', 'bool'),
332 332 ('use_gravatar', 'rhodecode_use_gravatar', 'bool'),
333 333 ('markup_renderer', 'rhodecode_markup_renderer', 'unicode'),
334 334 ('gravatar_url', 'rhodecode_gravatar_url', 'unicode'),
335 335 ('clone_uri_tmpl', 'rhodecode_clone_uri_tmpl', 'unicode'),
336 336 ('support_url', 'rhodecode_support_url', 'unicode'),
337 337 ('show_revision_number', 'rhodecode_show_revision_number', 'bool'),
338 338 ('show_sha_length', 'rhodecode_show_sha_length', 'int'),
339 339 ]
340 340 for setting, form_key, type_ in settings:
341 341 sett = SettingsModel().create_or_update_setting(
342 342 setting, form_result[form_key], type_)
343 343 Session().add(sett)
344 344
345 345 Session().commit()
346 346 SettingsModel().invalidate_settings_cache()
347 347 h.flash(_('Updated visualisation settings'), category='success')
348 348 except Exception:
349 349 log.exception("Exception updating visualization settings")
350 350 h.flash(_('Error occurred during updating '
351 351 'visualisation settings'),
352 352 category='error')
353 353
354 354 return redirect(url('admin_settings_visual'))
355 355
356 356 @HasPermissionAllDecorator('hg.admin')
357 357 def settings_visual(self):
358 358 """GET /admin/settings/visual: All items in the collection"""
359 359 # url('admin_settings_visual')
360 360 c.active = 'visual'
361 361
362 362 return htmlfill.render(
363 363 render('admin/settings/settings.html'),
364 364 defaults=self._form_defaults(),
365 365 encoding="UTF-8",
366 366 force_defaults=False)
367 367
368 368 @HasPermissionAllDecorator('hg.admin')
369 369 @auth.CSRFRequired()
370 370 def settings_issuetracker_test(self):
371 371 if request.is_xhr:
372 372 return h.urlify_commit_message(
373 373 request.POST.get('test_text', ''),
374 374 'repo_group/test_repo1')
375 375 else:
376 376 raise HTTPBadRequest()
377 377
378 378 @HasPermissionAllDecorator('hg.admin')
379 379 @auth.CSRFRequired()
380 380 def settings_issuetracker_delete(self):
381 381 uid = request.POST.get('uid')
382 382 IssueTrackerSettingsModel().delete_entries(uid)
383 383 h.flash(_('Removed issue tracker entry'), category='success')
384 384 return redirect(url('admin_settings_issuetracker'))
385 385
386 386 @HasPermissionAllDecorator('hg.admin')
387 387 def settings_issuetracker(self):
388 388 """GET /admin/settings/issue-tracker: All items in the collection"""
389 389 # url('admin_settings_issuetracker')
390 390 c.active = 'issuetracker'
391 391 defaults = SettingsModel().get_all_settings()
392 392
393 393 entry_key = 'rhodecode_issuetracker_pat_'
394 394
395 395 c.issuetracker_entries = {}
396 396 for k, v in defaults.items():
397 397 if k.startswith(entry_key):
398 398 uid = k[len(entry_key):]
399 399 c.issuetracker_entries[uid] = None
400 400
401 401 for uid in c.issuetracker_entries:
402 402 c.issuetracker_entries[uid] = AttributeDict({
403 403 'pat': defaults.get('rhodecode_issuetracker_pat_' + uid),
404 404 'url': defaults.get('rhodecode_issuetracker_url_' + uid),
405 405 'pref': defaults.get('rhodecode_issuetracker_pref_' + uid),
406 406 'desc': defaults.get('rhodecode_issuetracker_desc_' + uid),
407 407 })
408 408
409 409 return render('admin/settings/settings.html')
410 410
411 411 @HasPermissionAllDecorator('hg.admin')
412 412 @auth.CSRFRequired()
413 413 def settings_issuetracker_save(self):
414 414 settings_model = IssueTrackerSettingsModel()
415 415
416 416 form = IssueTrackerPatternsForm()().to_python(request.POST)
417 417 if form:
418 418 for uid in form.get('delete_patterns', []):
419 419 settings_model.delete_entries(uid)
420 420
421 421 for pattern in form.get('patterns', []):
422 422 for setting, value, type_ in pattern:
423 423 sett = settings_model.create_or_update_setting(
424 424 setting, value, type_)
425 425 Session().add(sett)
426 426
427 427 Session().commit()
428 428
429 429 SettingsModel().invalidate_settings_cache()
430 430 h.flash(_('Updated issue tracker entries'), category='success')
431 431 return redirect(url('admin_settings_issuetracker'))
432 432
433 433 @HasPermissionAllDecorator('hg.admin')
434 434 @auth.CSRFRequired()
435 435 def settings_email_update(self):
436 436 """POST /admin/settings/email: All items in the collection"""
437 437 # url('admin_settings_email')
438 438 c.active = 'email'
439 439
440 440 test_email = request.POST.get('test_email')
441 441
442 442 if not test_email:
443 443 h.flash(_('Please enter email address'), category='error')
444 444 return redirect(url('admin_settings_email'))
445 445
446 446 email_kwargs = {
447 447 'date': datetime.datetime.now(),
448 448 'user': c.rhodecode_user,
449 449 'rhodecode_version': c.rhodecode_version
450 450 }
451 451
452 452 (subject, headers, email_body,
453 453 email_body_plaintext) = EmailNotificationModel().render_email(
454 454 EmailNotificationModel.TYPE_EMAIL_TEST, **email_kwargs)
455 455
456 456 recipients = [test_email] if test_email else None
457 457
458 458 run_task(tasks.send_email, recipients, subject,
459 459 email_body_plaintext, email_body)
460 460
461 461 h.flash(_('Send email task created'), category='success')
462 462 return redirect(url('admin_settings_email'))
463 463
464 464 @HasPermissionAllDecorator('hg.admin')
465 465 def settings_email(self):
466 466 """GET /admin/settings/email: All items in the collection"""
467 467 # url('admin_settings_email')
468 468 c.active = 'email'
469 469 c.rhodecode_ini = rhodecode.CONFIG
470 470
471 471 return htmlfill.render(
472 472 render('admin/settings/settings.html'),
473 473 defaults=self._form_defaults(),
474 474 encoding="UTF-8",
475 475 force_defaults=False)
476 476
477 477 @HasPermissionAllDecorator('hg.admin')
478 478 @auth.CSRFRequired()
479 479 def settings_hooks_update(self):
480 480 """POST or DELETE /admin/settings/hooks: All items in the collection"""
481 481 # url('admin_settings_hooks')
482 482 c.active = 'hooks'
483 483 if c.visual.allow_custom_hooks_settings:
484 484 ui_key = request.POST.get('new_hook_ui_key')
485 485 ui_value = request.POST.get('new_hook_ui_value')
486 486
487 487 hook_id = request.POST.get('hook_id')
488 488 new_hook = False
489 489
490 490 model = SettingsModel()
491 491 try:
492 492 if ui_value and ui_key:
493 493 model.create_or_update_hook(ui_key, ui_value)
494 494 h.flash(_('Added new hook'), category='success')
495 495 new_hook = True
496 496 elif hook_id:
497 497 RhodeCodeUi.delete(hook_id)
498 498 Session().commit()
499 499
500 500 # check for edits
501 501 update = False
502 502 _d = request.POST.dict_of_lists()
503 503 for k, v in zip(_d.get('hook_ui_key', []),
504 504 _d.get('hook_ui_value_new', [])):
505 505 model.create_or_update_hook(k, v)
506 506 update = True
507 507
508 508 if update and not new_hook:
509 509 h.flash(_('Updated hooks'), category='success')
510 510 Session().commit()
511 511 except Exception:
512 512 log.exception("Exception during hook creation")
513 513 h.flash(_('Error occurred during hook creation'),
514 514 category='error')
515 515
516 516 return redirect(url('admin_settings_hooks'))
517 517
518 518 @HasPermissionAllDecorator('hg.admin')
519 519 def settings_hooks(self):
520 520 """GET /admin/settings/hooks: All items in the collection"""
521 521 # url('admin_settings_hooks')
522 522 c.active = 'hooks'
523 523
524 524 model = SettingsModel()
525 525 c.hooks = model.get_builtin_hooks()
526 526 c.custom_hooks = model.get_custom_hooks()
527 527
528 528 return htmlfill.render(
529 529 render('admin/settings/settings.html'),
530 530 defaults=self._form_defaults(),
531 531 encoding="UTF-8",
532 532 force_defaults=False)
533 533
534 534 @HasPermissionAllDecorator('hg.admin')
535 535 def settings_search(self):
536 536 """GET /admin/settings/search: All items in the collection"""
537 537 # url('admin_settings_search')
538 538 c.active = 'search'
539 539
540 540 from rhodecode.lib.index import searcher_from_config
541 541 searcher = searcher_from_config(config)
542 542 c.statistics = searcher.statistics()
543 543
544 544 return render('admin/settings/settings.html')
545 545
546 546 @HasPermissionAllDecorator('hg.admin')
547 547 def settings_system(self):
548 548 """GET /admin/settings/system: All items in the collection"""
549 549 # url('admin_settings_system')
550 550 snapshot = str2bool(request.GET.get('snapshot'))
551 c.active = 'system'
551 defaults = self._form_defaults()
552 552
553 defaults = self._form_defaults()
554 c.rhodecode_ini = rhodecode.CONFIG
553 c.active = 'system'
555 554 c.rhodecode_update_url = defaults.get('rhodecode_update_url')
556 555 server_info = ScmModel().get_server_info(request.environ)
556
557 557 for key, val in server_info.iteritems():
558 558 setattr(c, key, val)
559 559
560 if c.disk['percent'] > 90:
561 h.flash(h.literal(_(
562 'Critical: your disk space is very low <b>%s%%</b> used' %
563 c.disk['percent'])), 'error')
564 elif c.disk['percent'] > 70:
565 h.flash(h.literal(_(
566 'Warning: your disk space is running low <b>%s%%</b> used' %
567 c.disk['percent'])), 'warning')
560 def val(name, subkey='human_value'):
561 return server_info[name][subkey]
562
563 def state(name):
564 return server_info[name]['state']
565
566 def val2(name):
567 val = server_info[name]['human_value']
568 state = server_info[name]['state']
569 return val, state
568 570
569 try:
570 c.uptime_age = h._age(
571 h.time_to_datetime(c.boot_time), False, show_suffix=False)
572 except TypeError:
573 c.uptime_age = c.boot_time
571 c.data_items = [
572 # update info
573 (_('Update info'), h.literal(
574 '<span class="link" id="check_for_update" >%s.</span>' % (
575 _('Check for updates')) +
576 '<br/> <span >%s.</span>' % (_('Note: please make sure this server can access `%s` for the update link to work') % c.rhodecode_update_url)
577 ), ''),
578
579 # RhodeCode specific
580 (_('RhodeCode Version'), c.rhodecode_version, ''),
581 (_('RhodeCode Server IP'), val('server')['server_ip'], state('server')),
582 (_('RhodeCode Server ID'), val('server')['server_id'], state('server')),
583 (_('RhodeCode Configuration'), val('rhodecode_config')['path'], state('rhodecode_config')),
584 ('', '', ''), # spacer
585
586 # Database
587 (_('Database'), val('database')['url'], state('database')),
588 (_('Database version'), val('database')['version'], state('database')),
589 ('', '', ''), # spacer
574 590
575 try:
576 c.system_memory = '%s/%s, %s%% (%s%%) used%s' % (
577 h.format_byte_size_binary(c.memory['used']),
578 h.format_byte_size_binary(c.memory['total']),
579 c.memory['percent2'],
580 c.memory['percent'],
581 ' %s' % c.memory['error'] if 'error' in c.memory else '')
582 except TypeError:
583 c.system_memory = 'NOT AVAILABLE'
591 # Platform/Python
592 (_('Platform'), val('platform'), state('platform')),
593 (_('Python version'), val('python')['version'], state('python')),
594 (_('Python path'), val('python')['executable'], state('python')),
595 ('', '', ''), # spacer
596
597 # Systems stats
598 (_('CPU'), val('cpu'), state('cpu')),
599 (_('Load'), val('load'), state('load')),
600 (_('Memory'), val('memory'), state('memory')),
601 (_('Uptime'), val('uptime')['uptime'], state('uptime')),
602 ('', '', ''), # spacer
603
604 # Repo storage
605 (_('Storage location'), val('storage')['path'], state('storage')),
606 (_('Storage info'), val('storage')['text'], state('storage')),
607 (_('Storage inodes'), val('storage_inodes')['text'], state('storage_inodes')),
584 608
585 rhodecode_ini_safe = rhodecode.CONFIG.copy()
586 blacklist = [
587 'rhodecode_license_key',
588 'routes.map',
589 'pylons.h',
590 'pylons.app_globals',
591 'pylons.environ_config',
592 'sqlalchemy.db1.url',
593 ('app_conf', 'sqlalchemy.db1.url')
609 (_('Gist storage location'), val('storage_gist')['path'], state('storage_gist')),
610 (_('Gist storage info'), val('storage_gist')['text'], state('storage_gist')),
611
612 (_('Archive cache storage location'), val('storage_archive')['path'], state('storage_archive')),
613 (_('Archive cache info'), val('storage_archive')['text'], state('storage_archive')),
614
615 (_('Search storage'), val('storage_search')['path'], state('storage_search')),
616 (_('Search info'), val('storage_search')['text'], state('storage_search')),
617 ('', '', ''), # spacer
618
619 # VCS specific
620 (_('VCS Backends'), val('vcs_backends'), state('vcs_backends')),
621 (_('VCS Server'), val('vcs_server')['text'], state('vcs_server')),
622 (_('GIT'), val('git'), state('git')),
623 (_('HG'), val('hg'), state('hg')),
624 (_('SVN'), val('svn'), state('svn')),
625
594 626 ]
595 for k in blacklist:
596 if isinstance(k, tuple):
597 section, key = k
598 if section in rhodecode_ini_safe:
599 rhodecode_ini_safe[section].pop(key, None)
600 else:
601 rhodecode_ini_safe.pop(k, None)
602
603 c.rhodecode_ini_safe = rhodecode_ini_safe
604 627
605 628 # TODO: marcink, figure out how to allow only selected users to do this
606 c.allowed_to_snapshot = False
629 c.allowed_to_snapshot = c.rhodecode_user.admin
607 630
608 631 if snapshot:
609 632 if c.allowed_to_snapshot:
610 633 return render('admin/settings/settings_system_snapshot.html')
611 634 else:
612 635 h.flash('You are not allowed to do this', category='warning')
613 636
614 637 return htmlfill.render(
615 638 render('admin/settings/settings.html'),
616 639 defaults=defaults,
617 640 encoding="UTF-8",
618 641 force_defaults=False)
619 642
620 643 @staticmethod
621 644 def get_update_data(update_url):
622 645 """Return the JSON update data."""
623 646 ver = rhodecode.__version__
624 647 log.debug('Checking for upgrade on `%s` server', update_url)
625 648 opener = urllib2.build_opener()
626 649 opener.addheaders = [('User-agent', 'RhodeCode-SCM/%s' % ver)]
627 650 response = opener.open(update_url)
628 651 response_data = response.read()
629 652 data = json.loads(response_data)
630 653
631 654 return data
632 655
633 656 @HasPermissionAllDecorator('hg.admin')
634 657 def settings_system_update(self):
635 658 """GET /admin/settings/system/updates: All items in the collection"""
636 659 # url('admin_settings_system_update')
637 660 defaults = self._form_defaults()
638 661 update_url = defaults.get('rhodecode_update_url', '')
639 662
640 663 _err = lambda s: '<div style="color:#ff8888; padding:4px 0px">%s</div>' % (s)
641 664 try:
642 665 data = self.get_update_data(update_url)
643 666 except urllib2.URLError as e:
644 667 log.exception("Exception contacting upgrade server")
645 668 return _err('Failed to contact upgrade server: %r' % e)
646 669 except ValueError as e:
647 670 log.exception("Bad data sent from update server")
648 671 return _err('Bad data sent from update server')
649 672
650 673 latest = data['versions'][0]
651 674
652 675 c.update_url = update_url
653 676 c.latest_data = latest
654 677 c.latest_ver = latest['version']
655 678 c.cur_ver = rhodecode.__version__
656 679 c.should_upgrade = False
657 680
658 681 if (packaging.version.Version(c.latest_ver) >
659 682 packaging.version.Version(c.cur_ver)):
660 683 c.should_upgrade = True
661 684 c.important_notices = latest['general']
662 685
663 686 return render('admin/settings/settings_system_update.html')
664 687
665 688 @HasPermissionAllDecorator('hg.admin')
666 689 def settings_supervisor(self):
667 690 c.rhodecode_ini = rhodecode.CONFIG
668 691 c.active = 'supervisor'
669 692
670 693 c.supervisor_procs = OrderedDict([
671 694 (SUPERVISOR_MASTER, {}),
672 695 ])
673 696
674 697 c.log_size = 10240
675 698 supervisor = SupervisorModel()
676 699
677 700 _connection = supervisor.get_connection(
678 701 c.rhodecode_ini.get('supervisor.uri'))
679 702 c.connection_error = None
680 703 try:
681 704 _connection.supervisor.getAllProcessInfo()
682 705 except Exception as e:
683 706 c.connection_error = str(e)
684 707 log.exception("Exception reading supervisor data")
685 708 return render('admin/settings/settings.html')
686 709
687 710 groupid = c.rhodecode_ini.get('supervisor.group_id')
688 711
689 712 # feed our group processes to the main
690 713 for proc in supervisor.get_group_processes(_connection, groupid):
691 714 c.supervisor_procs[proc['name']] = {}
692 715
693 716 for k in c.supervisor_procs.keys():
694 717 try:
695 718 # master process info
696 719 if k == SUPERVISOR_MASTER:
697 720 _data = supervisor.get_master_state(_connection)
698 721 _data['name'] = 'supervisor master'
699 722 _data['description'] = 'pid %s, id: %s, ver: %s' % (
700 723 _data['pid'], _data['id'], _data['ver'])
701 724 c.supervisor_procs[k] = _data
702 725 else:
703 726 procid = groupid + ":" + k
704 727 c.supervisor_procs[k] = supervisor.get_process_info(_connection, procid)
705 728 except Exception as e:
706 729 log.exception("Exception reading supervisor data")
707 730 c.supervisor_procs[k] = {'_rhodecode_error': str(e)}
708 731
709 732 return render('admin/settings/settings.html')
710 733
711 734 @HasPermissionAllDecorator('hg.admin')
712 735 def settings_supervisor_log(self, procid):
713 736 import rhodecode
714 737 c.rhodecode_ini = rhodecode.CONFIG
715 738 c.active = 'supervisor_tail'
716 739
717 740 supervisor = SupervisorModel()
718 741 _connection = supervisor.get_connection(c.rhodecode_ini.get('supervisor.uri'))
719 742 groupid = c.rhodecode_ini.get('supervisor.group_id')
720 743 procid = groupid + ":" + procid if procid != SUPERVISOR_MASTER else procid
721 744
722 745 c.log_size = 10240
723 746 offset = abs(safe_int(request.GET.get('offset', c.log_size))) * -1
724 747 c.log = supervisor.read_process_log(_connection, procid, offset, 0)
725 748
726 749 return render('admin/settings/settings.html')
727 750
728 751 @HasPermissionAllDecorator('hg.admin')
729 752 @auth.CSRFRequired()
730 753 def settings_labs_update(self):
731 754 """POST /admin/settings/labs: All items in the collection"""
732 755 # url('admin_settings/labs', method={'POST'})
733 756 c.active = 'labs'
734 757
735 758 application_form = LabsSettingsForm()()
736 759 try:
737 760 form_result = application_form.to_python(dict(request.POST))
738 761 except formencode.Invalid as errors:
739 762 h.flash(
740 763 _('Some form inputs contain invalid data.'),
741 764 category='error')
742 765 return htmlfill.render(
743 766 render('admin/settings/settings.html'),
744 767 defaults=errors.value,
745 768 errors=errors.error_dict or {},
746 769 prefix_error=False,
747 770 encoding='UTF-8',
748 771 force_defaults=False
749 772 )
750 773
751 774 try:
752 775 session = Session()
753 776 for setting in _LAB_SETTINGS:
754 777 setting_name = setting.key[len('rhodecode_'):]
755 778 sett = SettingsModel().create_or_update_setting(
756 779 setting_name, form_result[setting.key], setting.type)
757 780 session.add(sett)
758 781
759 782 except Exception:
760 783 log.exception('Exception while updating lab settings')
761 784 h.flash(_('Error occurred during updating labs settings'),
762 785 category='error')
763 786 else:
764 787 Session().commit()
765 788 SettingsModel().invalidate_settings_cache()
766 789 h.flash(_('Updated Labs settings'), category='success')
767 790 return redirect(url('admin_settings_labs'))
768 791
769 792 return htmlfill.render(
770 793 render('admin/settings/settings.html'),
771 794 defaults=self._form_defaults(),
772 795 encoding='UTF-8',
773 796 force_defaults=False)
774 797
775 798 @HasPermissionAllDecorator('hg.admin')
776 799 def settings_labs(self):
777 800 """GET /admin/settings/labs: All items in the collection"""
778 801 # url('admin_settings_labs')
779 802 if not c.labs_active:
780 803 redirect(url('admin_settings'))
781 804
782 805 c.active = 'labs'
783 806 c.lab_settings = _LAB_SETTINGS
784 807
785 808 return htmlfill.render(
786 809 render('admin/settings/settings.html'),
787 810 defaults=self._form_defaults(),
788 811 encoding='UTF-8',
789 812 force_defaults=False)
790 813
791 814 def _form_defaults(self):
792 815 defaults = SettingsModel().get_all_settings()
793 816 defaults.update(self._get_hg_ui_settings())
794 817 defaults.update({
795 818 'new_svn_branch': '',
796 819 'new_svn_tag': '',
797 820 })
798 821 return defaults
799 822
800 823
801 824 # :param key: name of the setting including the 'rhodecode_' prefix
802 825 # :param type: the RhodeCodeSetting type to use.
803 826 # :param group: the i18ned group in which we should dispaly this setting
804 827 # :param label: the i18ned label we should display for this setting
805 828 # :param help: the i18ned help we should dispaly for this setting
806 829 LabSetting = collections.namedtuple(
807 830 'LabSetting', ('key', 'type', 'group', 'label', 'help'))
808 831
809 832
810 833 # This list has to be kept in sync with the form
811 834 # rhodecode.model.forms.LabsSettingsForm.
812 835 _LAB_SETTINGS = [
813 836
814 837 ]
@@ -1,300 +1,304 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2014-2016 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 """
22 22 Various version Control System version lib (vcs) management abstraction layer
23 23 for Python. Build with server client architecture.
24 24 """
25 25
26 26
27 27 VERSION = (0, 5, 0, 'dev')
28 28
29 29 __version__ = '.'.join((str(each) for each in VERSION[:4]))
30 30
31 31 __all__ = [
32 32 'get_version', 'get_vcs_instance', 'get_backend',
33 33 'VCSError', 'RepositoryError', 'CommitError'
34 34 ]
35 35
36 36 import atexit
37 37 import logging
38 38 import subprocess32
39 39 import time
40 40 import urlparse
41 41 from cStringIO import StringIO
42 42
43 43 import Pyro4
44 44 from Pyro4.errors import CommunicationError
45 45
46 46 from rhodecode.lib.vcs.conf import settings
47 47 from rhodecode.lib.vcs.backends import get_vcs_instance, get_backend
48 48 from rhodecode.lib.vcs.exceptions import (
49 49 VCSError, RepositoryError, CommitError, VCSCommunicationError)
50 50
51 51 log = logging.getLogger(__name__)
52 52
53 53 # The pycurl library directly accesses C API functions and is not patched by
54 54 # gevent. This will potentially lead to deadlocks due to incompatibility to
55 55 # gevent. Therefore we check if gevent is active and import a gevent compatible
56 56 # wrapper in that case.
57 57 try:
58 58 from gevent import monkey
59 59 if monkey.is_module_patched('__builtin__'):
60 60 import geventcurl as pycurl
61 61 log.debug('Using gevent comapatible pycurl: %s', pycurl)
62 62 else:
63 63 import pycurl
64 64 except ImportError:
65 65 import pycurl
66 66
67 67
68 68 def get_version():
69 69 """
70 70 Returns shorter version (digit parts only) as string.
71 71 """
72 72 return '.'.join((str(each) for each in VERSION[:3]))
73 73
74 74
75 75 def connect_pyro4(server_and_port):
76 76 from rhodecode.lib.vcs import connection, client
77 77 from rhodecode.lib.middleware.utils import scm_app
78 78
79 79 git_remote = client.RequestScopeProxyFactory(
80 80 settings.pyro_remote(settings.PYRO_GIT, server_and_port))
81 81 hg_remote = client.RequestScopeProxyFactory(
82 82 settings.pyro_remote(settings.PYRO_HG, server_and_port))
83 83 svn_remote = client.RequestScopeProxyFactory(
84 84 settings.pyro_remote(settings.PYRO_SVN, server_and_port))
85 85
86 86 connection.Git = client.RepoMaker(proxy_factory=git_remote)
87 87 connection.Hg = client.RepoMaker(proxy_factory=hg_remote)
88 88 connection.Svn = client.RepoMaker(proxy_factory=svn_remote)
89 89
90 90 scm_app.GIT_REMOTE_WSGI = Pyro4.Proxy(
91 91 settings.pyro_remote(
92 92 settings.PYRO_GIT_REMOTE_WSGI, server_and_port))
93 93 scm_app.HG_REMOTE_WSGI = Pyro4.Proxy(
94 94 settings.pyro_remote(
95 95 settings.PYRO_HG_REMOTE_WSGI, server_and_port))
96 96
97 97 @atexit.register
98 98 def free_connection_resources():
99 99 connection.Git = None
100 100 connection.Hg = None
101 101 connection.Svn = None
102 connection.Service = None
102 103
103 104
104 105 def connect_http(server_and_port):
105 106 from rhodecode.lib.vcs import connection, client_http
106 107 from rhodecode.lib.middleware.utils import scm_app
107 108
108 109 session_factory = client_http.ThreadlocalSessionFactory()
109 110
110 111 connection.Git = client_http.RepoMaker(
111 112 server_and_port, '/git', session_factory)
112 113 connection.Hg = client_http.RepoMaker(
113 114 server_and_port, '/hg', session_factory)
114 115 connection.Svn = client_http.RepoMaker(
115 116 server_and_port, '/svn', session_factory)
117 connection.Service = client_http.ServiceConnection(
118 server_and_port, '/_service', session_factory)
116 119
117 120 scm_app.HG_REMOTE_WSGI = client_http.VcsHttpProxy(
118 121 server_and_port, '/proxy/hg')
119 122 scm_app.GIT_REMOTE_WSGI = client_http.VcsHttpProxy(
120 123 server_and_port, '/proxy/git')
121 124
122 125 @atexit.register
123 126 def free_connection_resources():
124 127 connection.Git = None
125 128 connection.Hg = None
126 129 connection.Svn = None
130 connection.Service = None
127 131
128 132
129 133 def connect_vcs(server_and_port, protocol):
130 134 """
131 135 Initializes the connection to the vcs server.
132 136
133 137 :param server_and_port: str, e.g. "localhost:9900"
134 138 :param protocol: str, "pyro4" or "http"
135 139 """
136 140 if protocol == 'pyro4':
137 141 connect_pyro4(server_and_port)
138 142 elif protocol == 'http':
139 143 connect_http(server_and_port)
140 144 else:
141 145 raise Exception('Invalid vcs server protocol "{}"'.format(protocol))
142 146
143 147
144 148 # TODO: johbo: This function should be moved into our test suite, there is
145 149 # no reason to support starting the vcsserver in Enterprise itself.
146 150 def start_vcs_server(server_and_port, protocol, log_level=None):
147 151 """
148 152 Starts the vcs server in a subprocess.
149 153 """
150 154 log.info('Starting VCSServer as a sub process with %s protocol', protocol)
151 155 if protocol == 'http':
152 156 return _start_http_vcs_server(server_and_port, log_level)
153 157 elif protocol == 'pyro4':
154 158 return _start_pyro4_vcs_server(server_and_port, log_level)
155 159 else:
156 160 raise Exception('Invalid vcs server protocol "{}"'.format(protocol))
157 161
158 162
159 163 def _start_pyro4_vcs_server(server_and_port, log_level=None):
160 164 _try_to_shutdown_running_server(server_and_port, protocol='pyro4')
161 165 host, port = server_and_port.rsplit(":", 1)
162 166 host = host.strip('[]')
163 167 args = [
164 168 'vcsserver', '--port', port, '--host', host, '--locale', 'en_US.UTF-8',
165 169 '--threadpool', '32']
166 170 if log_level:
167 171 args += ['--log-level', log_level]
168 172 proc = subprocess32.Popen(args)
169 173
170 174 def cleanup_server_process():
171 175 proc.kill()
172 176 atexit.register(cleanup_server_process)
173 177
174 178 server = create_vcsserver_proxy(server_and_port, protocol='pyro4')
175 179 _wait_until_vcs_server_is_reachable(server)
176 180
177 181
178 182 def _start_http_vcs_server(server_and_port, log_level=None):
179 183 # TODO: mikhail: shutdown if an http server already runs
180 184
181 185 host, port = server_and_port.rsplit(":", 1)
182 186 args = [
183 187 'pserve', 'rhodecode/tests/vcsserver_http.ini',
184 188 'http_port=%s' % (port, ), 'http_host=%s' % (host, )]
185 189 proc = subprocess32.Popen(args)
186 190
187 191 def cleanup_server_process():
188 192 proc.kill()
189 193 atexit.register(cleanup_server_process)
190 194
191 195 server = create_vcsserver_proxy(server_and_port, protocol='http')
192 196 _wait_until_vcs_server_is_reachable(server)
193 197
194 198
195 199 def _wait_until_vcs_server_is_reachable(server, timeout=40):
196 200 begin = time.time()
197 201 while (time.time() - begin) < timeout:
198 202 try:
199 203 server.ping()
200 204 return
201 205 except (VCSCommunicationError, CommunicationError, pycurl.error):
202 206 log.debug('VCSServer not started yet, retry to connect.')
203 207 time.sleep(0.5)
204 208 raise Exception(
205 209 'Starting the VCSServer failed or took more than {} '
206 210 'seconds.'.format(timeout))
207 211
208 212
209 213 def _try_to_shutdown_running_server(server_and_port, protocol):
210 214 server = create_vcsserver_proxy(server_and_port, protocol)
211 215 try:
212 216 server.shutdown()
213 217 except (CommunicationError, pycurl.error):
214 218 return
215 219
216 220 # TODO: Not sure why this is important, but without it the following start
217 221 # of the server fails.
218 222 server = create_vcsserver_proxy(server_and_port, protocol)
219 223 server.ping()
220 224
221 225
222 226 def create_vcsserver_proxy(server_and_port, protocol):
223 227 if protocol == 'pyro4':
224 228 return _create_vcsserver_proxy_pyro4(server_and_port)
225 229 elif protocol == 'http':
226 230 return _create_vcsserver_proxy_http(server_and_port)
227 231 else:
228 232 raise Exception('Invalid vcs server protocol "{}"'.format(protocol))
229 233
230 234
231 235 def _create_vcsserver_proxy_pyro4(server_and_port):
232 236 server = Pyro4.Proxy(
233 237 settings.pyro_remote(settings.PYRO_VCSSERVER, server_and_port))
234 238 return server
235 239
236 240
237 241 def _create_vcsserver_proxy_http(server_and_port):
238 242 from rhodecode.lib.vcs import client_http
239 243
240 244 session = _create_http_rpc_session()
241 245 url = urlparse.urljoin('http://%s' % server_and_port, '/server')
242 246 return client_http.RemoteObject(url, session)
243 247
244 248
245 249 class CurlSession(object):
246 250 """
247 251 Modeled so that it provides a subset of the requests interface.
248 252
249 253 This has been created so that it does only provide a minimal API for our
250 254 needs. The parts which it provides are based on the API of the library
251 255 `requests` which allows us to easily benchmark against it.
252 256
253 257 Please have a look at the class :class:`requests.Session` when you extend
254 258 it.
255 259 """
256 260
257 261 def __init__(self):
258 262 curl = pycurl.Curl()
259 263 # TODO: johbo: I did test with 7.19 of libcurl. This version has
260 264 # trouble with 100 - continue being set in the expect header. This
261 265 # can lead to massive performance drops, switching it off here.
262 266 curl.setopt(curl.HTTPHEADER, ["Expect:"])
263 267 curl.setopt(curl.TCP_NODELAY, True)
264 268 curl.setopt(curl.PROTOCOLS, curl.PROTO_HTTP)
265 269 self._curl = curl
266 270
267 271 def post(self, url, data, allow_redirects=False):
268 272 response_buffer = StringIO()
269 273
270 274 curl = self._curl
271 275 curl.setopt(curl.URL, url)
272 276 curl.setopt(curl.POST, True)
273 277 curl.setopt(curl.POSTFIELDS, data)
274 278 curl.setopt(curl.FOLLOWLOCATION, allow_redirects)
275 279 curl.setopt(curl.WRITEDATA, response_buffer)
276 280 curl.perform()
277 281
278 282 return CurlResponse(response_buffer)
279 283
280 284
281 285 class CurlResponse(object):
282 286 """
283 287 The response of a request, modeled after the requests API.
284 288
285 289 This class provides a subset of the response interface known from the
286 290 library `requests`. It is intentionally kept similar, so that we can use
287 291 `requests` as a drop in replacement for benchmarking purposes.
288 292 """
289 293
290 294 def __init__(self, response_buffer):
291 295 self._response_buffer = response_buffer
292 296
293 297 @property
294 298 def content(self):
295 299 return self._response_buffer.getvalue()
296 300
297 301
298 302 def _create_http_rpc_session():
299 303 session = CurlSession()
300 304 return session
@@ -1,78 +1,87 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2014-2016 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 """
22 22 VCS Backends module
23 23 """
24 24
25 25 import logging
26 26
27 27 from pprint import pformat
28 28
29 29 from rhodecode.lib.vcs.conf import settings
30 30 from rhodecode.lib.vcs.exceptions import VCSError
31 31 from rhodecode.lib.vcs.utils.helpers import get_scm
32 32 from rhodecode.lib.vcs.utils.imports import import_class
33 33
34 34
35 35 log = logging.getLogger(__name__)
36 36
37 37
38 38 def get_vcs_instance(repo_path, *args, **kwargs):
39 39 """
40 40 Given a path to a repository an instance of the corresponding vcs backend
41 41 repository class is created and returned. If no repository can be found
42 42 for the path it returns None. Arguments and keyword arguments are passed
43 43 to the vcs backend repository class.
44 44 """
45 45 try:
46 46 vcs_alias = get_scm(repo_path)[0]
47 47 log.debug(
48 48 'Creating instance of %s repository from %s', vcs_alias, repo_path)
49 49 backend = get_backend(vcs_alias)
50 50 except VCSError:
51 51 log.exception(
52 52 'Perhaps this repository is in db and not in '
53 53 'filesystem run rescan repositories with '
54 54 '"destroy old data" option from admin panel')
55 55 return None
56 56
57 57 return backend(repo_path=repo_path, *args, **kwargs)
58 58
59 59
60 60 def get_backend(alias):
61 61 """
62 62 Returns ``Repository`` class identified by the given alias or raises
63 63 VCSError if alias is not recognized or backend class cannot be imported.
64 64 """
65 65 if alias not in settings.BACKENDS:
66 66 raise VCSError(
67 67 "Given alias '%s' is not recognized! Allowed aliases:\n%s" %
68 68 (alias, pformat(settings.BACKENDS.keys())))
69 69 backend_path = settings.BACKENDS[alias]
70 70 klass = import_class(backend_path)
71 71 return klass
72 72
73 73
74 74 def get_supported_backends():
75 75 """
76 76 Returns list of aliases of supported backends.
77 77 """
78 78 return settings.BACKENDS.keys()
79
80
81 def get_vcsserver_version():
82 from rhodecode.lib.vcs import connection
83 data = connection.Service.get_vcsserver_service_data()
84 if data and 'version' in data:
85 return data['version']
86
87 return None
@@ -1,47 +1,49 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2014-2016 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 """
22 22 GIT module
23 23 """
24 24
25 25 import logging
26 26
27 27 from rhodecode.lib.vcs import connection
28 28 from rhodecode.lib.vcs.backends.git.repository import GitRepository
29 29 from rhodecode.lib.vcs.backends.git.commit import GitCommit
30 30 from rhodecode.lib.vcs.backends.git.inmemory import GitInMemoryCommit
31 31
32 32
33 33 log = logging.getLogger(__name__)
34 34
35 35
36 def discover_git_version():
36 def discover_git_version(raise_on_exc=False):
37 37 """
38 38 Returns the string as it was returned by running 'git --version'
39 39
40 40 It will return an empty string in case the connection is not initialized
41 41 or no vcsserver is available.
42 42 """
43 43 try:
44 44 return connection.Git.discover_git_version()
45 45 except Exception:
46 46 log.warning("Failed to discover the Git version", exc_info=True)
47 if raise_on_exc:
48 raise
47 49 return ''
@@ -1,27 +1,48 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2014-2016 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 """
22 22 HG module
23 23 """
24 import logging
24 25
26 from rhodecode.lib.vcs import connection
25 27 from rhodecode.lib.vcs.backends.hg.commit import MercurialCommit
26 28 from rhodecode.lib.vcs.backends.hg.inmemory import MercurialInMemoryCommit
27 29 from rhodecode.lib.vcs.backends.hg.repository import MercurialRepository
30
31
32 log = logging.getLogger(__name__)
33
34
35 def discover_hg_version(raise_on_exc=False):
36 """
37 Returns the string as it was returned by running 'git --version'
38
39 It will return an empty string in case the connection is not initialized
40 or no vcsserver is available.
41 """
42 try:
43 return connection.Hg.discover_hg_version()
44 except Exception:
45 log.warning("Failed to discover the HG version", exc_info=True)
46 if raise_on_exc:
47 raise
48 return ''
@@ -1,26 +1,47 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2014-2016 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 """
22 22 SVN module
23 23 """
24 import logging
24 25
26 from rhodecode.lib.vcs import connection
25 27 from rhodecode.lib.vcs.backends.svn.commit import SubversionCommit
26 28 from rhodecode.lib.vcs.backends.svn.repository import SubversionRepository
29
30
31 log = logging.getLogger(__name__)
32
33
34 def discover_svn_version(raise_on_exc=False):
35 """
36 Returns the string as it was returned by running 'git --version'
37
38 It will return an empty string in case the connection is not initialized
39 or no vcsserver is available.
40 """
41 try:
42 return connection.Svn.discover_svn_version()
43 except Exception:
44 log.warning("Failed to discover the SVN version", exc_info=True)
45 if raise_on_exc:
46 raise
47 return ''
@@ -1,35 +1,37 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2014-2016 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 """
22 22 Holds connection for remote server.
23 23 """
24 24
25 25
26 26 def _not_initialized(*args, **kwargs):
27 27 """Placeholder for objects which have to be initialized first."""
28 28 raise Exception(
29 29 "rhodecode.lib.vcs is not yet initialized. "
30 30 "Make sure `vcs.server` is enabled in your configuration.")
31 31
32 32 # TODO: figure out a nice default value for these things
33 Service = _not_initialized
34
33 35 Git = _not_initialized
34 36 Hg = _not_initialized
35 37 Svn = _not_initialized
@@ -1,1123 +1,911 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 """
22 22 Scm model for RhodeCode
23 23 """
24 24
25 25 import os.path
26 26 import re
27 27 import sys
28 import time
29 28 import traceback
30 29 import logging
31 30 import cStringIO
32 31 import pkg_resources
33 32
34 import pylons
35 33 from pylons.i18n.translation import _
36 34 from sqlalchemy import func
37 35 from zope.cachedescriptors.property import Lazy as LazyProperty
38 36
39 37 import rhodecode
40 38 from rhodecode.lib.vcs import get_backend
41 39 from rhodecode.lib.vcs.exceptions import RepositoryError, NodeNotChangedError
42 40 from rhodecode.lib.vcs.nodes import FileNode
43 41 from rhodecode.lib.vcs.backends.base import EmptyCommit
44 42 from rhodecode.lib import helpers as h
45 43
46 44 from rhodecode.lib.auth import (
47 45 HasRepoPermissionAny, HasRepoGroupPermissionAny,
48 46 HasUserGroupPermissionAny)
49 47 from rhodecode.lib.exceptions import NonRelativePathError, IMCCommitError
50 48 from rhodecode.lib import hooks_utils, caches
51 49 from rhodecode.lib.utils import (
52 50 get_filesystem_repos, action_logger, make_db_config)
53 from rhodecode.lib.utils2 import (
54 safe_str, safe_unicode, get_server_url, md5)
51 from rhodecode.lib.utils2 import (safe_str, safe_unicode)
52 from rhodecode.lib.system_info import get_system_info
55 53 from rhodecode.model import BaseModel
56 54 from rhodecode.model.db import (
57 55 Repository, CacheKey, UserFollowing, UserLog, User, RepoGroup,
58 PullRequest, DbMigrateVersion)
56 PullRequest)
59 57 from rhodecode.model.settings import VcsSettingsModel
60 58
61 59 log = logging.getLogger(__name__)
62 60
63 61
64 62 class UserTemp(object):
65 63 def __init__(self, user_id):
66 64 self.user_id = user_id
67 65
68 66 def __repr__(self):
69 67 return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id)
70 68
71 69
72 70 class RepoTemp(object):
73 71 def __init__(self, repo_id):
74 72 self.repo_id = repo_id
75 73
76 74 def __repr__(self):
77 75 return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id)
78 76
79 77
80 78 class SimpleCachedRepoList(object):
81 79 """
82 80 Lighter version of of iteration of repos without the scm initialisation,
83 81 and with cache usage
84 82 """
85 83 def __init__(self, db_repo_list, repos_path, order_by=None, perm_set=None):
86 84 self.db_repo_list = db_repo_list
87 85 self.repos_path = repos_path
88 86 self.order_by = order_by
89 87 self.reversed = (order_by or '').startswith('-')
90 88 if not perm_set:
91 89 perm_set = ['repository.read', 'repository.write',
92 90 'repository.admin']
93 91 self.perm_set = perm_set
94 92
95 93 def __len__(self):
96 94 return len(self.db_repo_list)
97 95
98 96 def __repr__(self):
99 97 return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
100 98
101 99 def __iter__(self):
102 100 for dbr in self.db_repo_list:
103 101 # check permission at this level
104 102 has_perm = HasRepoPermissionAny(*self.perm_set)(
105 103 dbr.repo_name, 'SimpleCachedRepoList check')
106 104 if not has_perm:
107 105 continue
108 106
109 107 tmp_d = {
110 108 'name': dbr.repo_name,
111 109 'dbrepo': dbr.get_dict(),
112 110 'dbrepo_fork': dbr.fork.get_dict() if dbr.fork else {}
113 111 }
114 112 yield tmp_d
115 113
116 114
117 115 class _PermCheckIterator(object):
118 116
119 117 def __init__(
120 118 self, obj_list, obj_attr, perm_set, perm_checker,
121 119 extra_kwargs=None):
122 120 """
123 121 Creates iterator from given list of objects, additionally
124 122 checking permission for them from perm_set var
125 123
126 124 :param obj_list: list of db objects
127 125 :param obj_attr: attribute of object to pass into perm_checker
128 126 :param perm_set: list of permissions to check
129 127 :param perm_checker: callable to check permissions against
130 128 """
131 129 self.obj_list = obj_list
132 130 self.obj_attr = obj_attr
133 131 self.perm_set = perm_set
134 132 self.perm_checker = perm_checker
135 133 self.extra_kwargs = extra_kwargs or {}
136 134
137 135 def __len__(self):
138 136 return len(self.obj_list)
139 137
140 138 def __repr__(self):
141 139 return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
142 140
143 141 def __iter__(self):
144 142 checker = self.perm_checker(*self.perm_set)
145 143 for db_obj in self.obj_list:
146 144 # check permission at this level
147 145 name = getattr(db_obj, self.obj_attr, None)
148 146 if not checker(name, self.__class__.__name__, **self.extra_kwargs):
149 147 continue
150 148
151 149 yield db_obj
152 150
153 151
154 152 class RepoList(_PermCheckIterator):
155 153
156 154 def __init__(self, db_repo_list, perm_set=None, extra_kwargs=None):
157 155 if not perm_set:
158 156 perm_set = [
159 157 'repository.read', 'repository.write', 'repository.admin']
160 158
161 159 super(RepoList, self).__init__(
162 160 obj_list=db_repo_list,
163 161 obj_attr='repo_name', perm_set=perm_set,
164 162 perm_checker=HasRepoPermissionAny,
165 163 extra_kwargs=extra_kwargs)
166 164
167 165
168 166 class RepoGroupList(_PermCheckIterator):
169 167
170 168 def __init__(self, db_repo_group_list, perm_set=None, extra_kwargs=None):
171 169 if not perm_set:
172 170 perm_set = ['group.read', 'group.write', 'group.admin']
173 171
174 172 super(RepoGroupList, self).__init__(
175 173 obj_list=db_repo_group_list,
176 174 obj_attr='group_name', perm_set=perm_set,
177 175 perm_checker=HasRepoGroupPermissionAny,
178 176 extra_kwargs=extra_kwargs)
179 177
180 178
181 179 class UserGroupList(_PermCheckIterator):
182 180
183 181 def __init__(self, db_user_group_list, perm_set=None, extra_kwargs=None):
184 182 if not perm_set:
185 183 perm_set = ['usergroup.read', 'usergroup.write', 'usergroup.admin']
186 184
187 185 super(UserGroupList, self).__init__(
188 186 obj_list=db_user_group_list,
189 187 obj_attr='users_group_name', perm_set=perm_set,
190 188 perm_checker=HasUserGroupPermissionAny,
191 189 extra_kwargs=extra_kwargs)
192 190
193 191
194 192 class ScmModel(BaseModel):
195 193 """
196 194 Generic Scm Model
197 195 """
198 196
199 197 @LazyProperty
200 198 def repos_path(self):
201 199 """
202 200 Gets the repositories root path from database
203 201 """
204 202
205 203 settings_model = VcsSettingsModel(sa=self.sa)
206 204 return settings_model.get_repos_location()
207 205
208 206 def repo_scan(self, repos_path=None):
209 207 """
210 208 Listing of repositories in given path. This path should not be a
211 209 repository itself. Return a dictionary of repository objects
212 210
213 211 :param repos_path: path to directory containing repositories
214 212 """
215 213
216 214 if repos_path is None:
217 215 repos_path = self.repos_path
218 216
219 217 log.info('scanning for repositories in %s', repos_path)
220 218
221 219 config = make_db_config()
222 220 config.set('extensions', 'largefiles', '')
223 221 repos = {}
224 222
225 223 for name, path in get_filesystem_repos(repos_path, recursive=True):
226 224 # name need to be decomposed and put back together using the /
227 225 # since this is internal storage separator for rhodecode
228 226 name = Repository.normalize_repo_name(name)
229 227
230 228 try:
231 229 if name in repos:
232 230 raise RepositoryError('Duplicate repository name %s '
233 231 'found in %s' % (name, path))
234 232 elif path[0] in rhodecode.BACKENDS:
235 233 klass = get_backend(path[0])
236 234 repos[name] = klass(path[1], config=config)
237 235 except OSError:
238 236 continue
239 237 log.debug('found %s paths with repositories', len(repos))
240 238 return repos
241 239
242 240 def get_repos(self, all_repos=None, sort_key=None):
243 241 """
244 242 Get all repositories from db and for each repo create it's
245 243 backend instance and fill that backed with information from database
246 244
247 245 :param all_repos: list of repository names as strings
248 246 give specific repositories list, good for filtering
249 247
250 248 :param sort_key: initial sorting of repositories
251 249 """
252 250 if all_repos is None:
253 251 all_repos = self.sa.query(Repository)\
254 252 .filter(Repository.group_id == None)\
255 253 .order_by(func.lower(Repository.repo_name)).all()
256 254 repo_iter = SimpleCachedRepoList(
257 255 all_repos, repos_path=self.repos_path, order_by=sort_key)
258 256 return repo_iter
259 257
260 258 def get_repo_groups(self, all_groups=None):
261 259 if all_groups is None:
262 260 all_groups = RepoGroup.query()\
263 261 .filter(RepoGroup.group_parent_id == None).all()
264 262 return [x for x in RepoGroupList(all_groups)]
265 263
266 264 def mark_for_invalidation(self, repo_name, delete=False):
267 265 """
268 266 Mark caches of this repo invalid in the database. `delete` flag
269 267 removes the cache entries
270 268
271 269 :param repo_name: the repo_name for which caches should be marked
272 270 invalid, or deleted
273 271 :param delete: delete the entry keys instead of setting bool
274 272 flag on them
275 273 """
276 274 CacheKey.set_invalidate(repo_name, delete=delete)
277 275 repo = Repository.get_by_repo_name(repo_name)
278 276
279 277 if repo:
280 278 config = repo._config
281 279 config.set('extensions', 'largefiles', '')
282 280 repo.update_commit_cache(config=config, cs_cache=None)
283 281 caches.clear_repo_caches(repo_name)
284 282
285 283 def toggle_following_repo(self, follow_repo_id, user_id):
286 284
287 285 f = self.sa.query(UserFollowing)\
288 286 .filter(UserFollowing.follows_repo_id == follow_repo_id)\
289 287 .filter(UserFollowing.user_id == user_id).scalar()
290 288
291 289 if f is not None:
292 290 try:
293 291 self.sa.delete(f)
294 292 action_logger(UserTemp(user_id),
295 293 'stopped_following_repo',
296 294 RepoTemp(follow_repo_id))
297 295 return
298 296 except Exception:
299 297 log.error(traceback.format_exc())
300 298 raise
301 299
302 300 try:
303 301 f = UserFollowing()
304 302 f.user_id = user_id
305 303 f.follows_repo_id = follow_repo_id
306 304 self.sa.add(f)
307 305
308 306 action_logger(UserTemp(user_id),
309 307 'started_following_repo',
310 308 RepoTemp(follow_repo_id))
311 309 except Exception:
312 310 log.error(traceback.format_exc())
313 311 raise
314 312
315 313 def toggle_following_user(self, follow_user_id, user_id):
316 314 f = self.sa.query(UserFollowing)\
317 315 .filter(UserFollowing.follows_user_id == follow_user_id)\
318 316 .filter(UserFollowing.user_id == user_id).scalar()
319 317
320 318 if f is not None:
321 319 try:
322 320 self.sa.delete(f)
323 321 return
324 322 except Exception:
325 323 log.error(traceback.format_exc())
326 324 raise
327 325
328 326 try:
329 327 f = UserFollowing()
330 328 f.user_id = user_id
331 329 f.follows_user_id = follow_user_id
332 330 self.sa.add(f)
333 331 except Exception:
334 332 log.error(traceback.format_exc())
335 333 raise
336 334
337 335 def is_following_repo(self, repo_name, user_id, cache=False):
338 336 r = self.sa.query(Repository)\
339 337 .filter(Repository.repo_name == repo_name).scalar()
340 338
341 339 f = self.sa.query(UserFollowing)\
342 340 .filter(UserFollowing.follows_repository == r)\
343 341 .filter(UserFollowing.user_id == user_id).scalar()
344 342
345 343 return f is not None
346 344
347 345 def is_following_user(self, username, user_id, cache=False):
348 346 u = User.get_by_username(username)
349 347
350 348 f = self.sa.query(UserFollowing)\
351 349 .filter(UserFollowing.follows_user == u)\
352 350 .filter(UserFollowing.user_id == user_id).scalar()
353 351
354 352 return f is not None
355 353
356 354 def get_followers(self, repo):
357 355 repo = self._get_repo(repo)
358 356
359 357 return self.sa.query(UserFollowing)\
360 358 .filter(UserFollowing.follows_repository == repo).count()
361 359
362 360 def get_forks(self, repo):
363 361 repo = self._get_repo(repo)
364 362 return self.sa.query(Repository)\
365 363 .filter(Repository.fork == repo).count()
366 364
367 365 def get_pull_requests(self, repo):
368 366 repo = self._get_repo(repo)
369 367 return self.sa.query(PullRequest)\
370 368 .filter(PullRequest.target_repo == repo)\
371 369 .filter(PullRequest.status != PullRequest.STATUS_CLOSED).count()
372 370
373 371 def mark_as_fork(self, repo, fork, user):
374 372 repo = self._get_repo(repo)
375 373 fork = self._get_repo(fork)
376 374 if fork and repo.repo_id == fork.repo_id:
377 375 raise Exception("Cannot set repository as fork of itself")
378 376
379 377 if fork and repo.repo_type != fork.repo_type:
380 378 raise RepositoryError(
381 379 "Cannot set repository as fork of repository with other type")
382 380
383 381 repo.fork = fork
384 382 self.sa.add(repo)
385 383 return repo
386 384
387 385 def pull_changes(self, repo, username):
388 386 dbrepo = self._get_repo(repo)
389 387 clone_uri = dbrepo.clone_uri
390 388 if not clone_uri:
391 389 raise Exception("This repository doesn't have a clone uri")
392 390
393 391 repo = dbrepo.scm_instance(cache=False)
394 392 # TODO: marcink fix this an re-enable since we need common logic
395 393 # for hg/git remove hooks so we don't trigger them on fetching
396 394 # commits from remote
397 395 repo.config.clear_section('hooks')
398 396
399 397 repo_name = dbrepo.repo_name
400 398 try:
401 399 # TODO: we need to make sure those operations call proper hooks !
402 400 repo.pull(clone_uri)
403 401
404 402 self.mark_for_invalidation(repo_name)
405 403 except Exception:
406 404 log.error(traceback.format_exc())
407 405 raise
408 406
409 407 def commit_change(self, repo, repo_name, commit, user, author, message,
410 408 content, f_path):
411 409 """
412 410 Commits changes
413 411
414 412 :param repo: SCM instance
415 413
416 414 """
417 415 user = self._get_user(user)
418 416
419 417 # decoding here will force that we have proper encoded values
420 418 # in any other case this will throw exceptions and deny commit
421 419 content = safe_str(content)
422 420 path = safe_str(f_path)
423 421 # message and author needs to be unicode
424 422 # proper backend should then translate that into required type
425 423 message = safe_unicode(message)
426 424 author = safe_unicode(author)
427 425 imc = repo.in_memory_commit
428 426 imc.change(FileNode(path, content, mode=commit.get_file_mode(f_path)))
429 427 try:
430 428 # TODO: handle pre-push action !
431 429 tip = imc.commit(
432 430 message=message, author=author, parents=[commit],
433 431 branch=commit.branch)
434 432 except Exception as e:
435 433 log.error(traceback.format_exc())
436 434 raise IMCCommitError(str(e))
437 435 finally:
438 436 # always clear caches, if commit fails we want fresh object also
439 437 self.mark_for_invalidation(repo_name)
440 438
441 439 # We trigger the post-push action
442 440 hooks_utils.trigger_post_push_hook(
443 441 username=user.username, action='push_local', repo_name=repo_name,
444 442 repo_alias=repo.alias, commit_ids=[tip.raw_id])
445 443 return tip
446 444
447 445 def _sanitize_path(self, f_path):
448 446 if f_path.startswith('/') or f_path.startswith('./') or '../' in f_path:
449 447 raise NonRelativePathError('%s is not an relative path' % f_path)
450 448 if f_path:
451 449 f_path = os.path.normpath(f_path)
452 450 return f_path
453 451
454 452 def get_dirnode_metadata(self, commit, dir_node):
455 453 if not dir_node.is_dir():
456 454 return []
457 455
458 456 data = []
459 457 for node in dir_node:
460 458 if not node.is_file():
461 459 # we skip file-nodes
462 460 continue
463 461
464 462 last_commit = node.last_commit
465 463 last_commit_date = last_commit.date
466 464 data.append({
467 465 'name': node.name,
468 466 'size': h.format_byte_size_binary(node.size),
469 467 'modified_at': h.format_date(last_commit_date),
470 468 'modified_ts': last_commit_date.isoformat(),
471 469 'revision': last_commit.revision,
472 470 'short_id': last_commit.short_id,
473 471 'message': h.escape(last_commit.message),
474 472 'author': h.escape(last_commit.author),
475 473 'user_profile': h.gravatar_with_user(last_commit.author),
476 474 })
477 475
478 476 return data
479 477
480 478 def get_nodes(self, repo_name, commit_id, root_path='/', flat=True,
481 479 extended_info=False, content=False, max_file_bytes=None):
482 480 """
483 481 recursive walk in root dir and return a set of all path in that dir
484 482 based on repository walk function
485 483
486 484 :param repo_name: name of repository
487 485 :param commit_id: commit id for which to list nodes
488 486 :param root_path: root path to list
489 487 :param flat: return as a list, if False returns a dict with description
490 488 :param max_file_bytes: will not return file contents over this limit
491 489
492 490 """
493 491 _files = list()
494 492 _dirs = list()
495 493 try:
496 494 _repo = self._get_repo(repo_name)
497 495 commit = _repo.scm_instance().get_commit(commit_id=commit_id)
498 496 root_path = root_path.lstrip('/')
499 497 for __, dirs, files in commit.walk(root_path):
500 498 for f in files:
501 499 _content = None
502 500 _data = f.unicode_path
503 501 over_size_limit = (max_file_bytes is not None
504 502 and f.size > max_file_bytes)
505 503
506 504 if not flat:
507 505 _data = {
508 506 "name": f.unicode_path,
509 507 "type": "file",
510 508 }
511 509 if extended_info:
512 510 _data.update({
513 511 "md5": f.md5,
514 512 "binary": f.is_binary,
515 513 "size": f.size,
516 514 "extension": f.extension,
517 515 "mimetype": f.mimetype,
518 516 "lines": f.lines()[0]
519 517 })
520 518
521 519 if content:
522 520 full_content = None
523 521 if not f.is_binary and not over_size_limit:
524 522 full_content = safe_str(f.content)
525 523
526 524 _data.update({
527 525 "content": full_content,
528 526 })
529 527 _files.append(_data)
530 528 for d in dirs:
531 529 _data = d.unicode_path
532 530 if not flat:
533 531 _data = {
534 532 "name": d.unicode_path,
535 533 "type": "dir",
536 534 }
537 535 if extended_info:
538 536 _data.update({
539 537 "md5": None,
540 538 "binary": None,
541 539 "size": None,
542 540 "extension": None,
543 541 })
544 542 if content:
545 543 _data.update({
546 544 "content": None
547 545 })
548 546 _dirs.append(_data)
549 547 except RepositoryError:
550 548 log.debug("Exception in get_nodes", exc_info=True)
551 549 raise
552 550
553 551 return _dirs, _files
554 552
555 553 def create_nodes(self, user, repo, message, nodes, parent_commit=None,
556 554 author=None, trigger_push_hook=True):
557 555 """
558 556 Commits given multiple nodes into repo
559 557
560 558 :param user: RhodeCode User object or user_id, the commiter
561 559 :param repo: RhodeCode Repository object
562 560 :param message: commit message
563 561 :param nodes: mapping {filename:{'content':content},...}
564 562 :param parent_commit: parent commit, can be empty than it's
565 563 initial commit
566 564 :param author: author of commit, cna be different that commiter
567 565 only for git
568 566 :param trigger_push_hook: trigger push hooks
569 567
570 568 :returns: new commited commit
571 569 """
572 570
573 571 user = self._get_user(user)
574 572 scm_instance = repo.scm_instance(cache=False)
575 573
576 574 processed_nodes = []
577 575 for f_path in nodes:
578 576 f_path = self._sanitize_path(f_path)
579 577 content = nodes[f_path]['content']
580 578 f_path = safe_str(f_path)
581 579 # decoding here will force that we have proper encoded values
582 580 # in any other case this will throw exceptions and deny commit
583 581 if isinstance(content, (basestring,)):
584 582 content = safe_str(content)
585 583 elif isinstance(content, (file, cStringIO.OutputType,)):
586 584 content = content.read()
587 585 else:
588 586 raise Exception('Content is of unrecognized type %s' % (
589 587 type(content)
590 588 ))
591 589 processed_nodes.append((f_path, content))
592 590
593 591 message = safe_unicode(message)
594 592 commiter = user.full_contact
595 593 author = safe_unicode(author) if author else commiter
596 594
597 595 imc = scm_instance.in_memory_commit
598 596
599 597 if not parent_commit:
600 598 parent_commit = EmptyCommit(alias=scm_instance.alias)
601 599
602 600 if isinstance(parent_commit, EmptyCommit):
603 601 # EmptyCommit means we we're editing empty repository
604 602 parents = None
605 603 else:
606 604 parents = [parent_commit]
607 605 # add multiple nodes
608 606 for path, content in processed_nodes:
609 607 imc.add(FileNode(path, content=content))
610 608 # TODO: handle pre push scenario
611 609 tip = imc.commit(message=message,
612 610 author=author,
613 611 parents=parents,
614 612 branch=parent_commit.branch)
615 613
616 614 self.mark_for_invalidation(repo.repo_name)
617 615 if trigger_push_hook:
618 616 hooks_utils.trigger_post_push_hook(
619 617 username=user.username, action='push_local',
620 618 repo_name=repo.repo_name, repo_alias=scm_instance.alias,
621 619 commit_ids=[tip.raw_id])
622 620 return tip
623 621
624 622 def update_nodes(self, user, repo, message, nodes, parent_commit=None,
625 623 author=None, trigger_push_hook=True):
626 624 user = self._get_user(user)
627 625 scm_instance = repo.scm_instance(cache=False)
628 626
629 627 message = safe_unicode(message)
630 628 commiter = user.full_contact
631 629 author = safe_unicode(author) if author else commiter
632 630
633 631 imc = scm_instance.in_memory_commit
634 632
635 633 if not parent_commit:
636 634 parent_commit = EmptyCommit(alias=scm_instance.alias)
637 635
638 636 if isinstance(parent_commit, EmptyCommit):
639 637 # EmptyCommit means we we're editing empty repository
640 638 parents = None
641 639 else:
642 640 parents = [parent_commit]
643 641
644 642 # add multiple nodes
645 643 for _filename, data in nodes.items():
646 644 # new filename, can be renamed from the old one, also sanitaze
647 645 # the path for any hack around relative paths like ../../ etc.
648 646 filename = self._sanitize_path(data['filename'])
649 647 old_filename = self._sanitize_path(_filename)
650 648 content = data['content']
651 649
652 650 filenode = FileNode(old_filename, content=content)
653 651 op = data['op']
654 652 if op == 'add':
655 653 imc.add(filenode)
656 654 elif op == 'del':
657 655 imc.remove(filenode)
658 656 elif op == 'mod':
659 657 if filename != old_filename:
660 658 # TODO: handle renames more efficient, needs vcs lib
661 659 # changes
662 660 imc.remove(filenode)
663 661 imc.add(FileNode(filename, content=content))
664 662 else:
665 663 imc.change(filenode)
666 664
667 665 try:
668 666 # TODO: handle pre push scenario
669 667 # commit changes
670 668 tip = imc.commit(message=message,
671 669 author=author,
672 670 parents=parents,
673 671 branch=parent_commit.branch)
674 672 except NodeNotChangedError:
675 673 raise
676 674 except Exception as e:
677 675 log.exception("Unexpected exception during call to imc.commit")
678 676 raise IMCCommitError(str(e))
679 677 finally:
680 678 # always clear caches, if commit fails we want fresh object also
681 679 self.mark_for_invalidation(repo.repo_name)
682 680
683 681 if trigger_push_hook:
684 682 hooks_utils.trigger_post_push_hook(
685 683 username=user.username, action='push_local',
686 684 repo_name=repo.repo_name, repo_alias=scm_instance.alias,
687 685 commit_ids=[tip.raw_id])
688 686
689 687 def delete_nodes(self, user, repo, message, nodes, parent_commit=None,
690 688 author=None, trigger_push_hook=True):
691 689 """
692 690 Deletes given multiple nodes into `repo`
693 691
694 692 :param user: RhodeCode User object or user_id, the committer
695 693 :param repo: RhodeCode Repository object
696 694 :param message: commit message
697 695 :param nodes: mapping {filename:{'content':content},...}
698 696 :param parent_commit: parent commit, can be empty than it's initial
699 697 commit
700 698 :param author: author of commit, cna be different that commiter only
701 699 for git
702 700 :param trigger_push_hook: trigger push hooks
703 701
704 702 :returns: new commit after deletion
705 703 """
706 704
707 705 user = self._get_user(user)
708 706 scm_instance = repo.scm_instance(cache=False)
709 707
710 708 processed_nodes = []
711 709 for f_path in nodes:
712 710 f_path = self._sanitize_path(f_path)
713 711 # content can be empty but for compatabilty it allows same dicts
714 712 # structure as add_nodes
715 713 content = nodes[f_path].get('content')
716 714 processed_nodes.append((f_path, content))
717 715
718 716 message = safe_unicode(message)
719 717 commiter = user.full_contact
720 718 author = safe_unicode(author) if author else commiter
721 719
722 720 imc = scm_instance.in_memory_commit
723 721
724 722 if not parent_commit:
725 723 parent_commit = EmptyCommit(alias=scm_instance.alias)
726 724
727 725 if isinstance(parent_commit, EmptyCommit):
728 726 # EmptyCommit means we we're editing empty repository
729 727 parents = None
730 728 else:
731 729 parents = [parent_commit]
732 730 # add multiple nodes
733 731 for path, content in processed_nodes:
734 732 imc.remove(FileNode(path, content=content))
735 733
736 734 # TODO: handle pre push scenario
737 735 tip = imc.commit(message=message,
738 736 author=author,
739 737 parents=parents,
740 738 branch=parent_commit.branch)
741 739
742 740 self.mark_for_invalidation(repo.repo_name)
743 741 if trigger_push_hook:
744 742 hooks_utils.trigger_post_push_hook(
745 743 username=user.username, action='push_local',
746 744 repo_name=repo.repo_name, repo_alias=scm_instance.alias,
747 745 commit_ids=[tip.raw_id])
748 746 return tip
749 747
750 748 def strip(self, repo, commit_id, branch):
751 749 scm_instance = repo.scm_instance(cache=False)
752 750 scm_instance.config.clear_section('hooks')
753 751 scm_instance.strip(commit_id, branch)
754 752 self.mark_for_invalidation(repo.repo_name)
755 753
756 754 def get_unread_journal(self):
757 755 return self.sa.query(UserLog).count()
758 756
759 757 def get_repo_landing_revs(self, repo=None):
760 758 """
761 759 Generates select option with tags branches and bookmarks (for hg only)
762 760 grouped by type
763 761
764 762 :param repo:
765 763 """
766 764
767 765 hist_l = []
768 766 choices = []
769 767 repo = self._get_repo(repo)
770 768 hist_l.append(['rev:tip', _('latest tip')])
771 769 choices.append('rev:tip')
772 770 if not repo:
773 771 return choices, hist_l
774 772
775 773 repo = repo.scm_instance()
776 774
777 775 branches_group = (
778 776 [(u'branch:%s' % safe_unicode(b), safe_unicode(b))
779 777 for b in repo.branches],
780 778 _("Branches"))
781 779 hist_l.append(branches_group)
782 780 choices.extend([x[0] for x in branches_group[0]])
783 781
784 782 if repo.alias == 'hg':
785 783 bookmarks_group = (
786 784 [(u'book:%s' % safe_unicode(b), safe_unicode(b))
787 785 for b in repo.bookmarks],
788 786 _("Bookmarks"))
789 787 hist_l.append(bookmarks_group)
790 788 choices.extend([x[0] for x in bookmarks_group[0]])
791 789
792 790 tags_group = (
793 791 [(u'tag:%s' % safe_unicode(t), safe_unicode(t))
794 792 for t in repo.tags],
795 793 _("Tags"))
796 794 hist_l.append(tags_group)
797 795 choices.extend([x[0] for x in tags_group[0]])
798 796
799 797 return choices, hist_l
800 798
801 799 def install_git_hook(self, repo, force_create=False):
802 800 """
803 801 Creates a rhodecode hook inside a git repository
804 802
805 803 :param repo: Instance of VCS repo
806 804 :param force_create: Create even if same name hook exists
807 805 """
808 806
809 807 loc = os.path.join(repo.path, 'hooks')
810 808 if not repo.bare:
811 809 loc = os.path.join(repo.path, '.git', 'hooks')
812 810 if not os.path.isdir(loc):
813 811 os.makedirs(loc, mode=0777)
814 812
815 813 tmpl_post = pkg_resources.resource_string(
816 814 'rhodecode', '/'.join(
817 815 ('config', 'hook_templates', 'git_post_receive.py.tmpl')))
818 816 tmpl_pre = pkg_resources.resource_string(
819 817 'rhodecode', '/'.join(
820 818 ('config', 'hook_templates', 'git_pre_receive.py.tmpl')))
821 819
822 820 for h_type, tmpl in [('pre', tmpl_pre), ('post', tmpl_post)]:
823 821 _hook_file = os.path.join(loc, '%s-receive' % h_type)
824 822 log.debug('Installing git hook in repo %s', repo)
825 823 _rhodecode_hook = _check_rhodecode_hook(_hook_file)
826 824
827 825 if _rhodecode_hook or force_create:
828 826 log.debug('writing %s hook file !', h_type)
829 827 try:
830 828 with open(_hook_file, 'wb') as f:
831 829 tmpl = tmpl.replace('_TMPL_', rhodecode.__version__)
832 830 tmpl = tmpl.replace('_ENV_', sys.executable)
833 831 f.write(tmpl)
834 832 os.chmod(_hook_file, 0755)
835 833 except IOError:
836 834 log.exception('error writing hook file %s', _hook_file)
837 835 else:
838 836 log.debug('skipping writing hook file')
839 837
840 838 def install_svn_hooks(self, repo, force_create=False):
841 839 """
842 840 Creates rhodecode hooks inside a svn repository
843 841
844 842 :param repo: Instance of VCS repo
845 843 :param force_create: Create even if same name hook exists
846 844 """
847 845 hooks_path = os.path.join(repo.path, 'hooks')
848 846 if not os.path.isdir(hooks_path):
849 847 os.makedirs(hooks_path)
850 848 post_commit_tmpl = pkg_resources.resource_string(
851 849 'rhodecode', '/'.join(
852 850 ('config', 'hook_templates', 'svn_post_commit_hook.py.tmpl')))
853 851 pre_commit_template = pkg_resources.resource_string(
854 852 'rhodecode', '/'.join(
855 853 ('config', 'hook_templates', 'svn_pre_commit_hook.py.tmpl')))
856 854 templates = {
857 855 'post-commit': post_commit_tmpl,
858 856 'pre-commit': pre_commit_template
859 857 }
860 858 for filename in templates:
861 859 _hook_file = os.path.join(hooks_path, filename)
862 860 _rhodecode_hook = _check_rhodecode_hook(_hook_file)
863 861 if _rhodecode_hook or force_create:
864 862 log.debug('writing %s hook file !', filename)
865 863 template = templates[filename]
866 864 try:
867 865 with open(_hook_file, 'wb') as f:
868 866 template = template.replace(
869 867 '_TMPL_', rhodecode.__version__)
870 868 template = template.replace('_ENV_', sys.executable)
871 869 f.write(template)
872 870 os.chmod(_hook_file, 0755)
873 871 except IOError:
874 872 log.exception('error writing hook file %s', filename)
875 873 else:
876 874 log.debug('skipping writing hook file')
877 875
878 876 def install_hooks(self, repo, repo_type):
879 877 if repo_type == 'git':
880 878 self.install_git_hook(repo)
881 879 elif repo_type == 'svn':
882 880 self.install_svn_hooks(repo)
883 881
884 882 def get_server_info(self, environ=None):
885 import platform
886 import rhodecode
887 import pkg_resources
888 from rhodecode.model.meta import Base as sql_base, Session
889 from sqlalchemy.engine import url
890 from rhodecode.lib.base import get_server_ip_addr, get_server_port
891 from rhodecode.lib.vcs.backends.git import discover_git_version
892 from rhodecode.model.gist import GIST_STORE_LOC
893
894 def percentage(part, whole):
895 whole = float(whole)
896 if whole > 0:
897 return 100 * float(part) / whole
898 return 0
899
900 try:
901 # cygwin cannot have yet psutil support.
902 import psutil
903 except ImportError:
904 psutil = None
905
906 environ = environ or {}
907 _NA = 'NOT AVAILABLE'
908 _memory = _NA
909 _uptime = _NA
910 _boot_time = _NA
911 _cpu = _NA
912 _disk = dict(percent=0, used=0, total=0, error='')
913 _disk_inodes = dict(percent=0, free=0, used=0, total=0, error='')
914 _load = {'1_min': _NA, '5_min': _NA, '15_min': _NA}
915
916 model = VcsSettingsModel()
917 storage_path = model.get_repos_location()
918 gist_storage_path = os.path.join(storage_path, GIST_STORE_LOC)
919 archive_storage_path = rhodecode.CONFIG.get('archive_cache_dir', '')
920 search_index_storage_path = rhodecode.CONFIG.get('search.location', '')
921
922 if psutil:
923 # disk storage
924 try:
925 _disk = dict(psutil.disk_usage(storage_path)._asdict())
926 except Exception as e:
927 log.exception('Failed to fetch disk info')
928 _disk = {'percent': 0, 'used': 0, 'total': 0, 'error': str(e)}
929
930 # disk inodes usage
931 try:
932 i_stat = os.statvfs(storage_path)
933
934 _disk_inodes['used'] = i_stat.f_ffree
935 _disk_inodes['free'] = i_stat.f_favail
936 _disk_inodes['total'] = i_stat.f_files
937 _disk_inodes['percent'] = percentage(
938 _disk_inodes['used'], _disk_inodes['total'])
939 except Exception as e:
940 log.exception('Failed to fetch disk inodes info')
941 _disk_inodes['error'] = str(e)
942
943 # memory
944 _memory = dict(psutil.virtual_memory()._asdict())
945 _memory['percent2'] = psutil._common.usage_percent(
946 (_memory['total'] - _memory['free']),
947 _memory['total'], 1)
948
949 # load averages
950 if hasattr(psutil.os, 'getloadavg'):
951 _load = dict(zip(
952 ['1_min', '5_min', '15_min'], psutil.os.getloadavg()))
953 _uptime = time.time() - psutil.boot_time()
954 _boot_time = psutil.boot_time()
955 _cpu = psutil.cpu_percent(0.5)
956
957 mods = dict([(p.project_name, p.version)
958 for p in pkg_resources.working_set])
959
960 def get_storage_size(storage_path):
961 sizes = []
962 for file_ in os.listdir(storage_path):
963 storage_file = os.path.join(storage_path, file_)
964 if os.path.isfile(storage_file):
965 try:
966 sizes.append(os.path.getsize(storage_file))
967 except OSError:
968 log.exception('Failed to get size of storage file %s',
969 storage_file)
970 pass
971
972 return sum(sizes)
973
974 # archive cache storage
975 _disk_archive = {'percent': 0, 'used': 0, 'total': 0}
976 try:
977 archive_storage_path_exists = os.path.isdir(
978 archive_storage_path)
979 if archive_storage_path and archive_storage_path_exists:
980 used = get_storage_size(archive_storage_path)
981 _disk_archive.update({
982 'used': used,
983 'total': used,
984 })
985 except Exception as e:
986 log.exception('failed to fetch archive cache storage')
987 _disk_archive['error'] = str(e)
988
989 # search index storage
990 _disk_index = {'percent': 0, 'used': 0, 'total': 0}
991 try:
992 search_index_storage_path_exists = os.path.isdir(
993 search_index_storage_path)
994 if search_index_storage_path_exists:
995 used = get_storage_size(search_index_storage_path)
996 _disk_index.update({
997 'percent': 100,
998 'used': used,
999 'total': used,
1000 })
1001 except Exception as e:
1002 log.exception('failed to fetch search index storage')
1003 _disk_index['error'] = str(e)
1004
1005 # gist storage
1006 _disk_gist = {'percent': 0, 'used': 0, 'total': 0, 'items': 0}
1007 try:
1008 items_count = 0
1009 used = 0
1010 for root, dirs, files in os.walk(safe_str(gist_storage_path)):
1011 if root == gist_storage_path:
1012 items_count = len(dirs)
1013
1014 for f in files:
1015 try:
1016 used += os.path.getsize(os.path.join(root, f))
1017 except OSError:
1018 pass
1019 _disk_gist.update({
1020 'percent': 100,
1021 'used': used,
1022 'total': used,
1023 'items': items_count
1024 })
1025 except Exception as e:
1026 log.exception('failed to fetch gist storage items')
1027 _disk_gist['error'] = str(e)
1028
1029 # GIT info
1030 git_ver = discover_git_version()
1031
1032 # SVN info
1033 # TODO: johbo: Add discover_svn_version to replace this code.
1034 try:
1035 import svn.core
1036 svn_ver = svn.core.SVN_VERSION
1037 except ImportError:
1038 svn_ver = None
1039
1040 # DB stuff
1041 db_info = url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url'])
1042 db_type = db_info.__to_string__()
1043 try:
1044 engine = sql_base.metadata.bind
1045 db_server_info = engine.dialect._get_server_version_info(
1046 Session.connection(bind=engine))
1047 db_version = '%s %s' % (db_info.drivername,
1048 '.'.join(map(str, db_server_info)))
1049 except Exception:
1050 log.exception('failed to fetch db version')
1051 db_version = '%s %s' % (db_info.drivername, '?')
1052
1053 db_migrate = DbMigrateVersion.query().filter(
1054 DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one()
1055 db_migrate_version = db_migrate.version
1056
1057 info = {
1058 'py_version': ' '.join(platform._sys_version()),
1059 'py_path': sys.executable,
1060 'py_modules': sorted(mods.items(), key=lambda k: k[0].lower()),
1061
1062 'platform': safe_unicode(platform.platform()),
1063 'storage': storage_path,
1064 'archive_storage': archive_storage_path,
1065 'index_storage': search_index_storage_path,
1066 'gist_storage': gist_storage_path,
1067
1068
1069 'db_type': db_type,
1070 'db_version': db_version,
1071 'db_migrate_version': db_migrate_version,
1072
1073 'rhodecode_version': rhodecode.__version__,
1074 'rhodecode_config_ini': rhodecode.CONFIG.get('__file__'),
1075 'server_ip': '%s:%s' % (
1076 get_server_ip_addr(environ, log_errors=False),
1077 get_server_port(environ)
1078 ),
1079 'server_id': rhodecode.CONFIG.get('instance_id'),
1080
1081 'git_version': safe_unicode(git_ver),
1082 'hg_version': mods.get('mercurial'),
1083 'svn_version': svn_ver,
1084
1085 'uptime': _uptime,
1086 'boot_time': _boot_time,
1087 'load': _load,
1088 'cpu': _cpu,
1089 'memory': _memory,
1090 'disk': _disk,
1091 'disk_inodes': _disk_inodes,
1092 'disk_archive': _disk_archive,
1093 'disk_gist': _disk_gist,
1094 'disk_index': _disk_index,
1095 }
1096 return info
883 server_info = get_system_info(environ)
884 return server_info
1097 885
1098 886
1099 887 def _check_rhodecode_hook(hook_path):
1100 888 """
1101 889 Check if the hook was created by RhodeCode
1102 890 """
1103 891 if not os.path.exists(hook_path):
1104 892 return True
1105 893
1106 894 log.debug('hook exists, checking if it is from rhodecode')
1107 895 hook_content = _read_hook(hook_path)
1108 896 matches = re.search(r'(?:RC_HOOK_VER)\s*=\s*(.*)', hook_content)
1109 897 if matches:
1110 898 try:
1111 899 version = matches.groups()[0]
1112 900 log.debug('got %s, it is rhodecode', version)
1113 901 return True
1114 902 except Exception:
1115 903 log.exception("Exception while reading the hook version.")
1116 904
1117 905 return False
1118 906
1119 907
1120 908 def _read_hook(hook_path):
1121 909 with open(hook_path, 'rb') as f:
1122 910 content = f.read()
1123 911 return content
@@ -1,90 +1,57 b''
1 <%
2 elems = [
3 ## general
4 (_('RhodeCode Enterprise version'), h.literal('%s <div class="link" id="check_for_update" >%s</div>' % (c.rhodecode_version, _('check for updates'))), ''),
5 (_('Upgrade info endpoint'), h.literal('%s <br/><span >%s.</span>' % (c.rhodecode_update_url, _('Note: please make sure this server can access this url'))), ''),
6 (_('Configuration INI file'), c.rhodecode_config_ini, ''),
7 ## systems stats
8 (_('RhodeCode Enterprise Server IP'), c.server_ip, ''),
9 (_('RhodeCode Enterprise Server ID'), c.server_id, ''),
10 (_('Platform'), c.platform, ''),
11 (_('Uptime'), c.uptime_age, ''),
12 (_('Storage location'), c.storage, ''),
13 (_('Storage disk space'), "%s/%s, %s%% used%s" % (h.format_byte_size_binary(c.disk['used']), h.format_byte_size_binary(c.disk['total']),(c.disk['percent']), ' %s' % c.disk['error'] if 'error' in c.disk else ''), ''),
14 (_('Storage file limit (inodes)'), "%s/%s, %.1f%% used%s" % (c.disk_inodes['used'], c.disk_inodes['total'],(c.disk_inodes['percent']), ' %s' % c.disk_inodes['error'] if 'error' in c.disk_inodes else ''), ''),
15
16 (_('Search index storage'), c.index_storage, ''),
17 (_('Search index size'), "%s %s" % (h.format_byte_size_binary(c.disk_index['used']), ' %s' % c.disk_index['error'] if 'error' in c.disk_index else ''), ''),
18
19 (_('Gist storage'), c.gist_storage, ''),
20 (_('Gist storage size'), "%s (%s items)%s" % (h.format_byte_size_binary(c.disk_gist['used']),c.disk_gist['items'], ' %s' % c.disk_gist['error'] if 'error' in c.disk_gist else ''), ''),
21
22 (_('Archive cache'), h.literal('%s <br/><span >%s.</span>' % (c.archive_storage, _('Enable this by setting archive_cache_dir=/path/to/cache option in the .ini file'))), ''),
23 (_('Archive cache size'), "%s%s" % (h.format_byte_size_binary(c.disk_archive['used']), ' %s' % c.disk_archive['error'] if 'error' in c.disk_archive else ''), ''),
24
25 (_('System memory'), c.system_memory, ''),
26 (_('CPU'), '%s %%' %(c.cpu), ''),
27 (_('Load'), '1min: %s, 5min: %s, 15min: %s' %(c.load['1_min'],c.load['5_min'],c.load['15_min']), ''),
28
29 ## rhodecode stuff
30 (_('Python version'), c.py_version, ''),
31 (_('Python path'), c.py_path, ''),
32 (_('GIT version'), c.git_version, ''),
33 (_('HG version'), c.hg_version, ''),
34 (_('SVN version'), c.svn_version, ''),
35 (_('Database'), "%s @ version: %s" % (c.db_type, c.db_migrate_version), ''),
36 (_('Database version'), c.db_version, ''),
37
38 ]
39 %>
40 1
41 2 <div id="update_notice" style="display: none; margin: -40px 0px 20px 0px">
42 3 <div>${_('Checking for updates...')}</div>
43 4 </div>
44 5
45 6
46 7 <div class="panel panel-default">
47 8 <div class="panel-heading">
48 9 <h3 class="panel-title">${_('System Info')}</h3>
49 10 % if c.allowed_to_snapshot:
50 <a href="${url('admin_settings_system', snapshot=1)}" class="panel-edit">${_('create snapshot')}</a>
11 <a href="${url('admin_settings_system', snapshot=1)}" class="panel-edit">${_('create summary snapshot')}</a>
51 12 % endif
52 13 </div>
53 14 <div class="panel-body">
54 15 <dl class="dl-horizontal settings">
55 %for dt, dd, tt in elems:
56 <dt>${dt}:</dt>
57 <dd title="${tt}">${dd}</dd>
16 % for dt, dd, warn in c.data_items:
17 <dt>${dt}${':' if dt else '---'}</dt>
18 <dd>${dd}${'' if dt else '---'}
19 % if warn and warn['message']:
20 <div class="alert-${warn['type']}">
21 <strong>${warn['message']}</strong>
22 </div>
23 % endif
24 </dd>
58 25 %endfor
59 26 </dl>
60 27 </div>
61 28 </div>
62 29
63 30 <div class="panel panel-default">
64 31 <div class="panel-heading">
65 32 <h3 class="panel-title">${_('Python Packages')}</h3>
66 33 </div>
67 34 <div class="panel-body">
68 35 <table class="table">
69 36 <colgroup>
70 37 <col class='label'>
71 38 <col class='content'>
72 39 </colgroup>
73 40 <tbody>
74 %for key, value in c.py_modules:
41 % for key, value in c.py_modules['human_value']:
75 42 <tr>
76 43 <td>${key}</td>
77 44 <td>${value}</td>
78 45 </tr>
79 46 %endfor
80 47 </tbody>
81 48 </table>
82 49 </div>
83 50 </div>
84 51
85 52 <script>
86 53 $('#check_for_update').click(function(e){
87 54 $('#update_notice').show();
88 55 $('#update_notice').load("${h.url('admin_settings_system_update',version=c.rhodecode_version, platform=c.platform)}");
89 56 })
90 57 </script>
@@ -1,75 +1,41 b''
1 <%
2 elems = [
3 ## general
4 (_('RhodeCode Enterprise version'), c.rhodecode_version, ''),
5 (_('Upgrade info endpoint'), c.rhodecode_update_url, ''),
6 (_('Configuration INI file'), c.rhodecode_config_ini, ''),
7 ## systems stats
8 (_('RhodeCode Enterprise Server IP'), c.server_ip, ''),
9 (_('RhodeCode Enterprise Server ID'), c.server_id, ''),
10 (_('Platform'), c.platform, ''),
11 (_('Uptime'), c.uptime_age, ''),
12 (_('Storage location'), c.storage, ''),
13 (_('Storage disk space'), "%s/%s, %s%% used%s" % (h.format_byte_size_binary(c.disk['used']), h.format_byte_size_binary(c.disk['total']),(c.disk['percent']), ' %s' % c.disk['error'] if 'error' in c.disk else ''), ''),
14
15 (_('Search index storage'), c.index_storage, ''),
16 (_('Search index size'), "%s %s" % (h.format_byte_size_binary(c.disk_index['used']), ' %s' % c.disk_index['error'] if 'error' in c.disk_index else ''), ''),
17
18 (_('Gist storage'), c.gist_storage, ''),
19 (_('Gist storage size'), "%s (%s items)%s" % (h.format_byte_size_binary(c.disk_gist['used']),c.disk_gist['items'], ' %s' % c.disk_gist['error'] if 'error' in c.disk_gist else ''), ''),
20
21 (_('Archive cache'), c.archive_storage, ''),
22 (_('Archive cache size'), "%s%s" % (h.format_byte_size_binary(c.disk_archive['used']), ' %s' % c.disk_archive['error'] if 'error' in c.disk_archive else ''), ''),
23
24 (_('System memory'), c.system_memory, ''),
25 (_('CPU'), '%s %%' %(c.cpu), ''),
26 (_('Load'), '1min: %s, 5min: %s, 15min: %s' %(c.load['1_min'],c.load['5_min'],c.load['15_min']), ''),
27
28 ## rhodecode stuff
29 (_('Python version'), c.py_version, ''),
30 (_('Python path'), c.py_path, ''),
31 (_('GIT version'), c.git_version, ''),
32 (_('HG version'), c.hg_version, ''),
33 (_('SVN version'), c.svn_version, ''),
34 (_('Database'), "%s @ version: %s" % (c.db_type, c.db_migrate_version), ''),
35 (_('Database version'), c.db_version, ''),
36
37 ]
38 %>
39 1
40 2 <pre>
41 3 SYSTEM INFO
42 4 -----------
43 5
44 % for dt, dd, tt in elems:
45 ${dt}: ${dd}
6 % for dt, dd, warn in c.data_items:
7 ${dt}${':' if dt else '---'}
8 ${dd}
9 % if warn and warn['message']:
10 ALERT_${warn['type'].upper()} ${warn['message']}
11 % endif
46 12 % endfor
47 13
48 14 PYTHON PACKAGES
49 15 ---------------
50 16
51 % for key, value in c.py_modules:
17 % for key, value in c.py_modules['human_value']:
52 18 ${key}: ${value}
53 19 % endfor
54 20
55 21 SYSTEM SETTINGS
56 22 ---------------
57 23
58 % for key, value in sorted(c.rhodecode_ini_safe.items()):
24 % for key, value in sorted(c.rhodecode_config['human_value'].items()):
59 25 % if isinstance(value, dict):
60 26
61 27 % for key2, value2 in value.items():
62 28 [${key}]${key2}: ${value2}
63 29 % endfor
64 30
65 31 % else:
66 32 ${key}: ${value}
67 33 % endif
68 34 % endfor
69 35
70 36 </pre>
71 37
72 38
73 39
74 40
75 41
General Comments 0
You need to be logged in to leave comments. Login now