##// END OF EJS Templates
compat: use py3 compatible configparser in all places.
marcink -
r2355:1176c026 default
parent child Browse files
Show More
@@ -1,207 +1,207 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2016-2017 RhodeCode GmbH
3 # Copyright (C) 2016-2017 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import os
21 import os
22 import re
22 import re
23 import logging
23 import logging
24 import datetime
24 import datetime
25 import ConfigParser
25 from pyramid.compat import configparser
26
26
27 from rhodecode.model.db import Session, User, UserSshKeys
27 from rhodecode.model.db import Session, User, UserSshKeys
28 from rhodecode.model.scm import ScmModel
28 from rhodecode.model.scm import ScmModel
29
29
30 from .hg import MercurialServer
30 from .hg import MercurialServer
31 from .git import GitServer
31 from .git import GitServer
32 from .svn import SubversionServer
32 from .svn import SubversionServer
33 log = logging.getLogger(__name__)
33 log = logging.getLogger(__name__)
34
34
35
35
36 class SshWrapper(object):
36 class SshWrapper(object):
37
37
38 def __init__(self, command, connection_info, mode,
38 def __init__(self, command, connection_info, mode,
39 user, user_id, key_id, shell, ini_path, env):
39 user, user_id, key_id, shell, ini_path, env):
40 self.command = command
40 self.command = command
41 self.connection_info = connection_info
41 self.connection_info = connection_info
42 self.mode = mode
42 self.mode = mode
43 self.user = user
43 self.user = user
44 self.user_id = user_id
44 self.user_id = user_id
45 self.key_id = key_id
45 self.key_id = key_id
46 self.shell = shell
46 self.shell = shell
47 self.ini_path = ini_path
47 self.ini_path = ini_path
48 self.env = env
48 self.env = env
49
49
50 self.config = self.parse_config(ini_path)
50 self.config = self.parse_config(ini_path)
51 self.server_impl = None
51 self.server_impl = None
52
52
53 def parse_config(self, config_path):
53 def parse_config(self, config_path):
54 parser = ConfigParser.ConfigParser()
54 parser = configparser.ConfigParser()
55 parser.read(config_path)
55 parser.read(config_path)
56 return parser
56 return parser
57
57
58 def update_key_access_time(self, key_id):
58 def update_key_access_time(self, key_id):
59 key = UserSshKeys().query().filter(
59 key = UserSshKeys().query().filter(
60 UserSshKeys.ssh_key_id == key_id).scalar()
60 UserSshKeys.ssh_key_id == key_id).scalar()
61 if key:
61 if key:
62 key.accessed_on = datetime.datetime.utcnow()
62 key.accessed_on = datetime.datetime.utcnow()
63 Session().add(key)
63 Session().add(key)
64 Session().commit()
64 Session().commit()
65 log.debug('Update key `%s` access time', key_id)
65 log.debug('Update key `%s` access time', key_id)
66
66
67 def get_connection_info(self):
67 def get_connection_info(self):
68 """
68 """
69 connection_info
69 connection_info
70
70
71 Identifies the client and server ends of the connection.
71 Identifies the client and server ends of the connection.
72 The variable contains four space-separated values: client IP address,
72 The variable contains four space-separated values: client IP address,
73 client port number, server IP address, and server port number.
73 client port number, server IP address, and server port number.
74 """
74 """
75 conn = dict(
75 conn = dict(
76 client_ip=None,
76 client_ip=None,
77 client_port=None,
77 client_port=None,
78 server_ip=None,
78 server_ip=None,
79 server_port=None,
79 server_port=None,
80 )
80 )
81
81
82 info = self.connection_info.split(' ')
82 info = self.connection_info.split(' ')
83 if len(info) == 4:
83 if len(info) == 4:
84 conn['client_ip'] = info[0]
84 conn['client_ip'] = info[0]
85 conn['client_port'] = info[1]
85 conn['client_port'] = info[1]
86 conn['server_ip'] = info[2]
86 conn['server_ip'] = info[2]
87 conn['server_port'] = info[3]
87 conn['server_port'] = info[3]
88
88
89 return conn
89 return conn
90
90
91 def get_repo_details(self, mode):
91 def get_repo_details(self, mode):
92 vcs_type = mode if mode in ['svn', 'hg', 'git'] else None
92 vcs_type = mode if mode in ['svn', 'hg', 'git'] else None
93 mode = mode
93 mode = mode
94 repo_name = None
94 repo_name = None
95
95
96 hg_pattern = r'^hg\s+\-R\s+(\S+)\s+serve\s+\-\-stdio$'
96 hg_pattern = r'^hg\s+\-R\s+(\S+)\s+serve\s+\-\-stdio$'
97 hg_match = re.match(hg_pattern, self.command)
97 hg_match = re.match(hg_pattern, self.command)
98 if hg_match is not None:
98 if hg_match is not None:
99 vcs_type = 'hg'
99 vcs_type = 'hg'
100 repo_name = hg_match.group(1).strip('/')
100 repo_name = hg_match.group(1).strip('/')
101 return vcs_type, repo_name, mode
101 return vcs_type, repo_name, mode
102
102
103 git_pattern = (
103 git_pattern = (
104 r'^git-(receive-pack|upload-pack)\s\'[/]?(\S+?)(|\.git)\'$')
104 r'^git-(receive-pack|upload-pack)\s\'[/]?(\S+?)(|\.git)\'$')
105 git_match = re.match(git_pattern, self.command)
105 git_match = re.match(git_pattern, self.command)
106 if git_match is not None:
106 if git_match is not None:
107 vcs_type = 'git'
107 vcs_type = 'git'
108 repo_name = git_match.group(2).strip('/')
108 repo_name = git_match.group(2).strip('/')
109 mode = git_match.group(1)
109 mode = git_match.group(1)
110 return vcs_type, repo_name, mode
110 return vcs_type, repo_name, mode
111
111
112 svn_pattern = r'^svnserve -t'
112 svn_pattern = r'^svnserve -t'
113 svn_match = re.match(svn_pattern, self.command)
113 svn_match = re.match(svn_pattern, self.command)
114
114
115 if svn_match is not None:
115 if svn_match is not None:
116 vcs_type = 'svn'
116 vcs_type = 'svn'
117 # Repo name should be extracted from the input stream
117 # Repo name should be extracted from the input stream
118 return vcs_type, repo_name, mode
118 return vcs_type, repo_name, mode
119
119
120 return vcs_type, repo_name, mode
120 return vcs_type, repo_name, mode
121
121
122 def serve(self, vcs, repo, mode, user, permissions):
122 def serve(self, vcs, repo, mode, user, permissions):
123 store = ScmModel().repos_path
123 store = ScmModel().repos_path
124
124
125 log.debug(
125 log.debug(
126 'VCS detected:`%s` mode: `%s` repo_name: %s', vcs, mode, repo)
126 'VCS detected:`%s` mode: `%s` repo_name: %s', vcs, mode, repo)
127
127
128 if vcs == 'hg':
128 if vcs == 'hg':
129 server = MercurialServer(
129 server = MercurialServer(
130 store=store, ini_path=self.ini_path,
130 store=store, ini_path=self.ini_path,
131 repo_name=repo, user=user,
131 repo_name=repo, user=user,
132 user_permissions=permissions, config=self.config, env=self.env)
132 user_permissions=permissions, config=self.config, env=self.env)
133 self.server_impl = server
133 self.server_impl = server
134 return server.run()
134 return server.run()
135
135
136 elif vcs == 'git':
136 elif vcs == 'git':
137 server = GitServer(
137 server = GitServer(
138 store=store, ini_path=self.ini_path,
138 store=store, ini_path=self.ini_path,
139 repo_name=repo, repo_mode=mode, user=user,
139 repo_name=repo, repo_mode=mode, user=user,
140 user_permissions=permissions, config=self.config, env=self.env)
140 user_permissions=permissions, config=self.config, env=self.env)
141 self.server_impl = server
141 self.server_impl = server
142 return server.run()
142 return server.run()
143
143
144 elif vcs == 'svn':
144 elif vcs == 'svn':
145 server = SubversionServer(
145 server = SubversionServer(
146 store=store, ini_path=self.ini_path,
146 store=store, ini_path=self.ini_path,
147 repo_name=None, user=user,
147 repo_name=None, user=user,
148 user_permissions=permissions, config=self.config, env=self.env)
148 user_permissions=permissions, config=self.config, env=self.env)
149 self.server_impl = server
149 self.server_impl = server
150 return server.run()
150 return server.run()
151
151
152 else:
152 else:
153 raise Exception('Unrecognised VCS: {}'.format(vcs))
153 raise Exception('Unrecognised VCS: {}'.format(vcs))
154
154
155 def wrap(self):
155 def wrap(self):
156 mode = self.mode
156 mode = self.mode
157 user = self.user
157 user = self.user
158 user_id = self.user_id
158 user_id = self.user_id
159 key_id = self.key_id
159 key_id = self.key_id
160 shell = self.shell
160 shell = self.shell
161
161
162 scm_detected, scm_repo, scm_mode = self.get_repo_details(mode)
162 scm_detected, scm_repo, scm_mode = self.get_repo_details(mode)
163
163
164 log.debug(
164 log.debug(
165 'Mode: `%s` User: `%s:%s` Shell: `%s` SSH Command: `\"%s\"` '
165 'Mode: `%s` User: `%s:%s` Shell: `%s` SSH Command: `\"%s\"` '
166 'SCM_DETECTED: `%s` SCM Mode: `%s` SCM Repo: `%s`',
166 'SCM_DETECTED: `%s` SCM Mode: `%s` SCM Repo: `%s`',
167 mode, user, user_id, shell, self.command,
167 mode, user, user_id, shell, self.command,
168 scm_detected, scm_mode, scm_repo)
168 scm_detected, scm_mode, scm_repo)
169
169
170 # update last access time for this key
170 # update last access time for this key
171 self.update_key_access_time(key_id)
171 self.update_key_access_time(key_id)
172
172
173 log.debug('SSH Connection info %s', self.get_connection_info())
173 log.debug('SSH Connection info %s', self.get_connection_info())
174
174
175 if shell and self.command is None:
175 if shell and self.command is None:
176 log.info(
176 log.info(
177 'Dropping to shell, no command given and shell is allowed')
177 'Dropping to shell, no command given and shell is allowed')
178 os.execl('/bin/bash', '-l')
178 os.execl('/bin/bash', '-l')
179 exit_code = 1
179 exit_code = 1
180
180
181 elif scm_detected:
181 elif scm_detected:
182 user = User.get(user_id)
182 user = User.get(user_id)
183 if not user:
183 if not user:
184 log.warning('User with id %s not found', user_id)
184 log.warning('User with id %s not found', user_id)
185 exit_code = -1
185 exit_code = -1
186 return exit_code
186 return exit_code
187
187
188 auth_user = user.AuthUser()
188 auth_user = user.AuthUser()
189 permissions = auth_user.permissions['repositories']
189 permissions = auth_user.permissions['repositories']
190
190
191 try:
191 try:
192 exit_code, is_updated = self.serve(
192 exit_code, is_updated = self.serve(
193 scm_detected, scm_repo, scm_mode, user, permissions)
193 scm_detected, scm_repo, scm_mode, user, permissions)
194 except Exception:
194 except Exception:
195 log.exception('Error occurred during execution of SshWrapper')
195 log.exception('Error occurred during execution of SshWrapper')
196 exit_code = -1
196 exit_code = -1
197
197
198 elif self.command is None and shell is False:
198 elif self.command is None and shell is False:
199 log.error('No Command given.')
199 log.error('No Command given.')
200 exit_code = -1
200 exit_code = -1
201
201
202 else:
202 else:
203 log.error(
203 log.error(
204 'Unhandled Command: "%s" Aborting.', self.command)
204 'Unhandled Command: "%s" Aborting.', self.command)
205 exit_code = -1
205 exit_code = -1
206
206
207 return exit_code
207 return exit_code
@@ -1,62 +1,62 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2016-2017 RhodeCode GmbH
3 # Copyright (C) 2016-2017 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import os
21 import os
22 import pytest
22 import pytest
23 import ConfigParser
23 from pyramid.compat import configparser
24
24
25 from rhodecode.apps.ssh_support.lib.ssh_wrapper import SshWrapper
25 from rhodecode.apps.ssh_support.lib.ssh_wrapper import SshWrapper
26 from rhodecode.lib.utils2 import AttributeDict
26 from rhodecode.lib.utils2 import AttributeDict
27
27
28
28
29 @pytest.fixture
29 @pytest.fixture
30 def dummy_conf_file(tmpdir):
30 def dummy_conf_file(tmpdir):
31 conf = ConfigParser.ConfigParser()
31 conf = configparser.ConfigParser()
32 conf.add_section('app:main')
32 conf.add_section('app:main')
33 conf.set('app:main', 'ssh.executable.hg', '/usr/bin/hg')
33 conf.set('app:main', 'ssh.executable.hg', '/usr/bin/hg')
34 conf.set('app:main', 'ssh.executable.git', '/usr/bin/git')
34 conf.set('app:main', 'ssh.executable.git', '/usr/bin/git')
35 conf.set('app:main', 'ssh.executable.svn', '/usr/bin/svnserve')
35 conf.set('app:main', 'ssh.executable.svn', '/usr/bin/svnserve')
36
36
37 f_path = os.path.join(str(tmpdir), 'ssh_wrapper_test.ini')
37 f_path = os.path.join(str(tmpdir), 'ssh_wrapper_test.ini')
38 with open(f_path, 'wb') as f:
38 with open(f_path, 'wb') as f:
39 conf.write(f)
39 conf.write(f)
40
40
41 return os.path.join(f_path)
41 return os.path.join(f_path)
42
42
43
43
44 @pytest.fixture
44 @pytest.fixture
45 def dummy_env():
45 def dummy_env():
46 return {
46 return {
47 'request':
47 'request':
48 AttributeDict(host_url='http://localhost', script_name='/')
48 AttributeDict(host_url='http://localhost', script_name='/')
49 }
49 }
50
50
51
51
52 @pytest.fixture
52 @pytest.fixture
53 def dummy_user():
53 def dummy_user():
54 return AttributeDict(username='test_user')
54 return AttributeDict(username='test_user')
55
55
56
56
57 @pytest.fixture
57 @pytest.fixture
58 def ssh_wrapper(app, dummy_conf_file, dummy_env):
58 def ssh_wrapper(app, dummy_conf_file, dummy_env):
59 conn_info = '127.0.0.1 22 10.0.0.1 443'
59 conn_info = '127.0.0.1 22 10.0.0.1 443'
60 return SshWrapper(
60 return SshWrapper(
61 'random command', conn_info, 'auto', 'admin', '1', key_id='1',
61 'random command', conn_info, 'auto', 'admin', '1', key_id='1',
62 shell=False, ini_path=dummy_conf_file, env=dummy_env)
62 shell=False, ini_path=dummy_conf_file, env=dummy_env)
@@ -1,49 +1,49 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2016-2017 RhodeCode GmbH
3 # Copyright (C) 2016-2017 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import os
21 import os
22 import ConfigParser
22 from pyramid.compat import configparser
23 from pyramid.paster import bootstrap as pyramid_bootstrap
23 from pyramid.paster import bootstrap as pyramid_bootstrap, setup_logging # noqa
24 from pyramid.request import Request
24 from pyramid.request import Request
25
25
26
26
27 def get_config(ini_path, **kwargs):
27 def get_config(ini_path, **kwargs):
28 parser = ConfigParser.ConfigParser(**kwargs)
28 parser = configparser.ConfigParser(**kwargs)
29 parser.read(ini_path)
29 parser.read(ini_path)
30 return parser
30 return parser
31
31
32
32
33 def get_app_config(ini_path):
33 def get_app_config(ini_path):
34 from paste.deploy.loadwsgi import appconfig
34 from paste.deploy.loadwsgi import appconfig
35 return appconfig('config:{}'.format(ini_path), relative_to=os.getcwd())
35 return appconfig('config:{}'.format(ini_path), relative_to=os.getcwd())
36
36
37
37
38 def bootstrap(config_uri, request=None, options=None):
38 def bootstrap(config_uri, request=None, options=None):
39
39
40 config = get_config(config_uri)
40 config = get_config(config_uri)
41 base_url = 'http://rhodecode.local'
41 base_url = 'http://rhodecode.local'
42 try:
42 try:
43 base_url = config.get('app:main', 'app.base_url')
43 base_url = config.get('app:main', 'app.base_url')
44 except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
44 except (configparser.NoSectionError, configparser.NoOptionError):
45 pass
45 pass
46
46
47 request = request or Request.blank('/', base_url=base_url)
47 request = request or Request.blank('/', base_url=base_url)
48
48
49 return pyramid_bootstrap(config_uri, request=request, options=options)
49 return pyramid_bootstrap(config_uri, request=request, options=options)
@@ -1,723 +1,722 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2017-2017 RhodeCode GmbH
3 # Copyright (C) 2017-2017 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21
21
22 import os
22 import os
23 import sys
23 import sys
24 import time
24 import time
25 import platform
25 import platform
26 import pkg_resources
26 import pkg_resources
27 import logging
27 import logging
28 import string
29
28
29 from pyramid.compat import configparser
30
30
31 log = logging.getLogger(__name__)
31 log = logging.getLogger(__name__)
32
32
33
33
34 psutil = None
34 psutil = None
35
35
36 try:
36 try:
37 # cygwin cannot have yet psutil support.
37 # cygwin cannot have yet psutil support.
38 import psutil as psutil
38 import psutil as psutil
39 except ImportError:
39 except ImportError:
40 pass
40 pass
41
41
42
42
43 _NA = 'NOT AVAILABLE'
43 _NA = 'NOT AVAILABLE'
44
44
45 STATE_OK = 'ok'
45 STATE_OK = 'ok'
46 STATE_ERR = 'error'
46 STATE_ERR = 'error'
47 STATE_WARN = 'warning'
47 STATE_WARN = 'warning'
48
48
49 STATE_OK_DEFAULT = {'message': '', 'type': STATE_OK}
49 STATE_OK_DEFAULT = {'message': '', 'type': STATE_OK}
50
50
51
51
52 # HELPERS
52 # HELPERS
53 def percentage(part, whole):
53 def percentage(part, whole):
54 whole = float(whole)
54 whole = float(whole)
55 if whole > 0:
55 if whole > 0:
56 return round(100 * float(part) / whole, 1)
56 return round(100 * float(part) / whole, 1)
57 return 0.0
57 return 0.0
58
58
59
59
60 def get_storage_size(storage_path):
60 def get_storage_size(storage_path):
61 sizes = []
61 sizes = []
62 for file_ in os.listdir(storage_path):
62 for file_ in os.listdir(storage_path):
63 storage_file = os.path.join(storage_path, file_)
63 storage_file = os.path.join(storage_path, file_)
64 if os.path.isfile(storage_file):
64 if os.path.isfile(storage_file):
65 try:
65 try:
66 sizes.append(os.path.getsize(storage_file))
66 sizes.append(os.path.getsize(storage_file))
67 except OSError:
67 except OSError:
68 log.exception('Failed to get size of storage file %s',
68 log.exception('Failed to get size of storage file %s',
69 storage_file)
69 storage_file)
70 pass
70 pass
71
71
72 return sum(sizes)
72 return sum(sizes)
73
73
74
74
75 class SysInfoRes(object):
75 class SysInfoRes(object):
76 def __init__(self, value, state=STATE_OK_DEFAULT, human_value=None):
76 def __init__(self, value, state=STATE_OK_DEFAULT, human_value=None):
77 self.value = value
77 self.value = value
78 self.state = state
78 self.state = state
79 self.human_value = human_value or value
79 self.human_value = human_value or value
80
80
81 def __json__(self):
81 def __json__(self):
82 return {
82 return {
83 'value': self.value,
83 'value': self.value,
84 'state': self.state,
84 'state': self.state,
85 'human_value': self.human_value,
85 'human_value': self.human_value,
86 }
86 }
87
87
88 def get_value(self):
88 def get_value(self):
89 return self.__json__()
89 return self.__json__()
90
90
91 def __str__(self):
91 def __str__(self):
92 return '<SysInfoRes({})>'.format(self.__json__())
92 return '<SysInfoRes({})>'.format(self.__json__())
93
93
94
94
95 class SysInfo(object):
95 class SysInfo(object):
96
96
97 def __init__(self, func_name, **kwargs):
97 def __init__(self, func_name, **kwargs):
98 self.func_name = func_name
98 self.func_name = func_name
99 self.value = _NA
99 self.value = _NA
100 self.state = None
100 self.state = None
101 self.kwargs = kwargs or {}
101 self.kwargs = kwargs or {}
102
102
103 def __call__(self):
103 def __call__(self):
104 computed = self.compute(**self.kwargs)
104 computed = self.compute(**self.kwargs)
105 if not isinstance(computed, SysInfoRes):
105 if not isinstance(computed, SysInfoRes):
106 raise ValueError(
106 raise ValueError(
107 'computed value for {} is not instance of '
107 'computed value for {} is not instance of '
108 '{}, got {} instead'.format(
108 '{}, got {} instead'.format(
109 self.func_name, SysInfoRes, type(computed)))
109 self.func_name, SysInfoRes, type(computed)))
110 return computed.__json__()
110 return computed.__json__()
111
111
112 def __str__(self):
112 def __str__(self):
113 return '<SysInfo({})>'.format(self.func_name)
113 return '<SysInfo({})>'.format(self.func_name)
114
114
115 def compute(self, **kwargs):
115 def compute(self, **kwargs):
116 return self.func_name(**kwargs)
116 return self.func_name(**kwargs)
117
117
118
118
119 # SysInfo functions
119 # SysInfo functions
120 def python_info():
120 def python_info():
121 value = dict(version=' '.join(platform._sys_version()),
121 value = dict(version=' '.join(platform._sys_version()),
122 executable=sys.executable)
122 executable=sys.executable)
123 return SysInfoRes(value=value)
123 return SysInfoRes(value=value)
124
124
125
125
126 def py_modules():
126 def py_modules():
127 mods = dict([(p.project_name, p.version)
127 mods = dict([(p.project_name, p.version)
128 for p in pkg_resources.working_set])
128 for p in pkg_resources.working_set])
129 value = sorted(mods.items(), key=lambda k: k[0].lower())
129 value = sorted(mods.items(), key=lambda k: k[0].lower())
130 return SysInfoRes(value=value)
130 return SysInfoRes(value=value)
131
131
132
132
133 def platform_type():
133 def platform_type():
134 from rhodecode.lib.utils import safe_unicode, generate_platform_uuid
134 from rhodecode.lib.utils import safe_unicode, generate_platform_uuid
135
135
136 value = dict(
136 value = dict(
137 name=safe_unicode(platform.platform()),
137 name=safe_unicode(platform.platform()),
138 uuid=generate_platform_uuid()
138 uuid=generate_platform_uuid()
139 )
139 )
140 return SysInfoRes(value=value)
140 return SysInfoRes(value=value)
141
141
142
142
143 def uptime():
143 def uptime():
144 from rhodecode.lib.helpers import age, time_to_datetime
144 from rhodecode.lib.helpers import age, time_to_datetime
145 from rhodecode.translation import TranslationString
145 from rhodecode.translation import TranslationString
146
146
147 value = dict(boot_time=0, uptime=0, text='')
147 value = dict(boot_time=0, uptime=0, text='')
148 state = STATE_OK_DEFAULT
148 state = STATE_OK_DEFAULT
149 if not psutil:
149 if not psutil:
150 return SysInfoRes(value=value, state=state)
150 return SysInfoRes(value=value, state=state)
151
151
152 boot_time = psutil.boot_time()
152 boot_time = psutil.boot_time()
153 value['boot_time'] = boot_time
153 value['boot_time'] = boot_time
154 value['uptime'] = time.time() - boot_time
154 value['uptime'] = time.time() - boot_time
155
155
156 date_or_age = age(time_to_datetime(boot_time))
156 date_or_age = age(time_to_datetime(boot_time))
157 if isinstance(date_or_age, TranslationString):
157 if isinstance(date_or_age, TranslationString):
158 date_or_age = date_or_age.interpolate()
158 date_or_age = date_or_age.interpolate()
159
159
160 human_value = value.copy()
160 human_value = value.copy()
161 human_value['boot_time'] = time_to_datetime(boot_time)
161 human_value['boot_time'] = time_to_datetime(boot_time)
162 human_value['uptime'] = age(time_to_datetime(boot_time), show_suffix=False)
162 human_value['uptime'] = age(time_to_datetime(boot_time), show_suffix=False)
163
163
164 human_value['text'] = u'Server started {}'.format(date_or_age)
164 human_value['text'] = u'Server started {}'.format(date_or_age)
165 return SysInfoRes(value=value, human_value=human_value)
165 return SysInfoRes(value=value, human_value=human_value)
166
166
167
167
168 def memory():
168 def memory():
169 from rhodecode.lib.helpers import format_byte_size_binary
169 from rhodecode.lib.helpers import format_byte_size_binary
170 value = dict(available=0, used=0, used_real=0, cached=0, percent=0,
170 value = dict(available=0, used=0, used_real=0, cached=0, percent=0,
171 percent_used=0, free=0, inactive=0, active=0, shared=0,
171 percent_used=0, free=0, inactive=0, active=0, shared=0,
172 total=0, buffers=0, text='')
172 total=0, buffers=0, text='')
173
173
174 state = STATE_OK_DEFAULT
174 state = STATE_OK_DEFAULT
175 if not psutil:
175 if not psutil:
176 return SysInfoRes(value=value, state=state)
176 return SysInfoRes(value=value, state=state)
177
177
178 value.update(dict(psutil.virtual_memory()._asdict()))
178 value.update(dict(psutil.virtual_memory()._asdict()))
179 value['used_real'] = value['total'] - value['available']
179 value['used_real'] = value['total'] - value['available']
180 value['percent_used'] = psutil._common.usage_percent(
180 value['percent_used'] = psutil._common.usage_percent(
181 value['used_real'], value['total'], 1)
181 value['used_real'], value['total'], 1)
182
182
183 human_value = value.copy()
183 human_value = value.copy()
184 human_value['text'] = '%s/%s, %s%% used' % (
184 human_value['text'] = '%s/%s, %s%% used' % (
185 format_byte_size_binary(value['used_real']),
185 format_byte_size_binary(value['used_real']),
186 format_byte_size_binary(value['total']),
186 format_byte_size_binary(value['total']),
187 value['percent_used'],)
187 value['percent_used'],)
188
188
189 keys = value.keys()[::]
189 keys = value.keys()[::]
190 keys.pop(keys.index('percent'))
190 keys.pop(keys.index('percent'))
191 keys.pop(keys.index('percent_used'))
191 keys.pop(keys.index('percent_used'))
192 keys.pop(keys.index('text'))
192 keys.pop(keys.index('text'))
193 for k in keys:
193 for k in keys:
194 human_value[k] = format_byte_size_binary(value[k])
194 human_value[k] = format_byte_size_binary(value[k])
195
195
196 if state['type'] == STATE_OK and value['percent_used'] > 90:
196 if state['type'] == STATE_OK and value['percent_used'] > 90:
197 msg = 'Critical: your available RAM memory is very low.'
197 msg = 'Critical: your available RAM memory is very low.'
198 state = {'message': msg, 'type': STATE_ERR}
198 state = {'message': msg, 'type': STATE_ERR}
199
199
200 elif state['type'] == STATE_OK and value['percent_used'] > 70:
200 elif state['type'] == STATE_OK and value['percent_used'] > 70:
201 msg = 'Warning: your available RAM memory is running low.'
201 msg = 'Warning: your available RAM memory is running low.'
202 state = {'message': msg, 'type': STATE_WARN}
202 state = {'message': msg, 'type': STATE_WARN}
203
203
204 return SysInfoRes(value=value, state=state, human_value=human_value)
204 return SysInfoRes(value=value, state=state, human_value=human_value)
205
205
206
206
207 def machine_load():
207 def machine_load():
208 value = {'1_min': _NA, '5_min': _NA, '15_min': _NA, 'text': ''}
208 value = {'1_min': _NA, '5_min': _NA, '15_min': _NA, 'text': ''}
209 state = STATE_OK_DEFAULT
209 state = STATE_OK_DEFAULT
210 if not psutil:
210 if not psutil:
211 return SysInfoRes(value=value, state=state)
211 return SysInfoRes(value=value, state=state)
212
212
213 # load averages
213 # load averages
214 if hasattr(psutil.os, 'getloadavg'):
214 if hasattr(psutil.os, 'getloadavg'):
215 value.update(dict(
215 value.update(dict(
216 zip(['1_min', '5_min', '15_min'], psutil.os.getloadavg())))
216 zip(['1_min', '5_min', '15_min'], psutil.os.getloadavg())))
217
217
218 human_value = value.copy()
218 human_value = value.copy()
219 human_value['text'] = '1min: {}, 5min: {}, 15min: {}'.format(
219 human_value['text'] = '1min: {}, 5min: {}, 15min: {}'.format(
220 value['1_min'], value['5_min'], value['15_min'])
220 value['1_min'], value['5_min'], value['15_min'])
221
221
222 if state['type'] == STATE_OK and value['15_min'] > 5:
222 if state['type'] == STATE_OK and value['15_min'] > 5:
223 msg = 'Warning: your machine load is very high.'
223 msg = 'Warning: your machine load is very high.'
224 state = {'message': msg, 'type': STATE_WARN}
224 state = {'message': msg, 'type': STATE_WARN}
225
225
226 return SysInfoRes(value=value, state=state, human_value=human_value)
226 return SysInfoRes(value=value, state=state, human_value=human_value)
227
227
228
228
229 def cpu():
229 def cpu():
230 value = {'cpu': 0, 'cpu_count': 0, 'cpu_usage': []}
230 value = {'cpu': 0, 'cpu_count': 0, 'cpu_usage': []}
231 state = STATE_OK_DEFAULT
231 state = STATE_OK_DEFAULT
232
232
233 if not psutil:
233 if not psutil:
234 return SysInfoRes(value=value, state=state)
234 return SysInfoRes(value=value, state=state)
235
235
236 value['cpu'] = psutil.cpu_percent(0.5)
236 value['cpu'] = psutil.cpu_percent(0.5)
237 value['cpu_usage'] = psutil.cpu_percent(0.5, percpu=True)
237 value['cpu_usage'] = psutil.cpu_percent(0.5, percpu=True)
238 value['cpu_count'] = psutil.cpu_count()
238 value['cpu_count'] = psutil.cpu_count()
239
239
240 human_value = value.copy()
240 human_value = value.copy()
241 human_value['text'] = '{} cores at {} %'.format(
241 human_value['text'] = '{} cores at {} %'.format(
242 value['cpu_count'], value['cpu'])
242 value['cpu_count'], value['cpu'])
243
243
244 return SysInfoRes(value=value, state=state, human_value=human_value)
244 return SysInfoRes(value=value, state=state, human_value=human_value)
245
245
246
246
247 def storage():
247 def storage():
248 from rhodecode.lib.helpers import format_byte_size_binary
248 from rhodecode.lib.helpers import format_byte_size_binary
249 from rhodecode.model.settings import VcsSettingsModel
249 from rhodecode.model.settings import VcsSettingsModel
250 path = VcsSettingsModel().get_repos_location()
250 path = VcsSettingsModel().get_repos_location()
251
251
252 value = dict(percent=0, used=0, total=0, path=path, text='')
252 value = dict(percent=0, used=0, total=0, path=path, text='')
253 state = STATE_OK_DEFAULT
253 state = STATE_OK_DEFAULT
254 if not psutil:
254 if not psutil:
255 return SysInfoRes(value=value, state=state)
255 return SysInfoRes(value=value, state=state)
256
256
257 try:
257 try:
258 value.update(dict(psutil.disk_usage(path)._asdict()))
258 value.update(dict(psutil.disk_usage(path)._asdict()))
259 except Exception as e:
259 except Exception as e:
260 log.exception('Failed to fetch disk info')
260 log.exception('Failed to fetch disk info')
261 state = {'message': str(e), 'type': STATE_ERR}
261 state = {'message': str(e), 'type': STATE_ERR}
262
262
263 human_value = value.copy()
263 human_value = value.copy()
264 human_value['used'] = format_byte_size_binary(value['used'])
264 human_value['used'] = format_byte_size_binary(value['used'])
265 human_value['total'] = format_byte_size_binary(value['total'])
265 human_value['total'] = format_byte_size_binary(value['total'])
266 human_value['text'] = "{}/{}, {}% used".format(
266 human_value['text'] = "{}/{}, {}% used".format(
267 format_byte_size_binary(value['used']),
267 format_byte_size_binary(value['used']),
268 format_byte_size_binary(value['total']),
268 format_byte_size_binary(value['total']),
269 value['percent'])
269 value['percent'])
270
270
271 if state['type'] == STATE_OK and value['percent'] > 90:
271 if state['type'] == STATE_OK and value['percent'] > 90:
272 msg = 'Critical: your disk space is very low.'
272 msg = 'Critical: your disk space is very low.'
273 state = {'message': msg, 'type': STATE_ERR}
273 state = {'message': msg, 'type': STATE_ERR}
274
274
275 elif state['type'] == STATE_OK and value['percent'] > 70:
275 elif state['type'] == STATE_OK and value['percent'] > 70:
276 msg = 'Warning: your disk space is running low.'
276 msg = 'Warning: your disk space is running low.'
277 state = {'message': msg, 'type': STATE_WARN}
277 state = {'message': msg, 'type': STATE_WARN}
278
278
279 return SysInfoRes(value=value, state=state, human_value=human_value)
279 return SysInfoRes(value=value, state=state, human_value=human_value)
280
280
281
281
282 def storage_inodes():
282 def storage_inodes():
283 from rhodecode.model.settings import VcsSettingsModel
283 from rhodecode.model.settings import VcsSettingsModel
284 path = VcsSettingsModel().get_repos_location()
284 path = VcsSettingsModel().get_repos_location()
285
285
286 value = dict(percent=0, free=0, used=0, total=0, path=path, text='')
286 value = dict(percent=0, free=0, used=0, total=0, path=path, text='')
287 state = STATE_OK_DEFAULT
287 state = STATE_OK_DEFAULT
288 if not psutil:
288 if not psutil:
289 return SysInfoRes(value=value, state=state)
289 return SysInfoRes(value=value, state=state)
290
290
291 try:
291 try:
292 i_stat = os.statvfs(path)
292 i_stat = os.statvfs(path)
293 value['free'] = i_stat.f_ffree
293 value['free'] = i_stat.f_ffree
294 value['used'] = i_stat.f_files-i_stat.f_favail
294 value['used'] = i_stat.f_files-i_stat.f_favail
295 value['total'] = i_stat.f_files
295 value['total'] = i_stat.f_files
296 value['percent'] = percentage(value['used'], value['total'])
296 value['percent'] = percentage(value['used'], value['total'])
297 except Exception as e:
297 except Exception as e:
298 log.exception('Failed to fetch disk inodes info')
298 log.exception('Failed to fetch disk inodes info')
299 state = {'message': str(e), 'type': STATE_ERR}
299 state = {'message': str(e), 'type': STATE_ERR}
300
300
301 human_value = value.copy()
301 human_value = value.copy()
302 human_value['text'] = "{}/{}, {}% used".format(
302 human_value['text'] = "{}/{}, {}% used".format(
303 value['used'], value['total'], value['percent'])
303 value['used'], value['total'], value['percent'])
304
304
305 if state['type'] == STATE_OK and value['percent'] > 90:
305 if state['type'] == STATE_OK and value['percent'] > 90:
306 msg = 'Critical: your disk free inodes are very low.'
306 msg = 'Critical: your disk free inodes are very low.'
307 state = {'message': msg, 'type': STATE_ERR}
307 state = {'message': msg, 'type': STATE_ERR}
308
308
309 elif state['type'] == STATE_OK and value['percent'] > 70:
309 elif state['type'] == STATE_OK and value['percent'] > 70:
310 msg = 'Warning: your disk free inodes are running low.'
310 msg = 'Warning: your disk free inodes are running low.'
311 state = {'message': msg, 'type': STATE_WARN}
311 state = {'message': msg, 'type': STATE_WARN}
312
312
313 return SysInfoRes(value=value, state=state, human_value=human_value)
313 return SysInfoRes(value=value, state=state, human_value=human_value)
314
314
315
315
316 def storage_archives():
316 def storage_archives():
317 import rhodecode
317 import rhodecode
318 from rhodecode.lib.utils import safe_str
318 from rhodecode.lib.utils import safe_str
319 from rhodecode.lib.helpers import format_byte_size_binary
319 from rhodecode.lib.helpers import format_byte_size_binary
320
320
321 msg = 'Enable this by setting ' \
321 msg = 'Enable this by setting ' \
322 'archive_cache_dir=/path/to/cache option in the .ini file'
322 'archive_cache_dir=/path/to/cache option in the .ini file'
323 path = safe_str(rhodecode.CONFIG.get('archive_cache_dir', msg))
323 path = safe_str(rhodecode.CONFIG.get('archive_cache_dir', msg))
324
324
325 value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
325 value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
326 state = STATE_OK_DEFAULT
326 state = STATE_OK_DEFAULT
327 try:
327 try:
328 items_count = 0
328 items_count = 0
329 used = 0
329 used = 0
330 for root, dirs, files in os.walk(path):
330 for root, dirs, files in os.walk(path):
331 if root == path:
331 if root == path:
332 items_count = len(files)
332 items_count = len(files)
333
333
334 for f in files:
334 for f in files:
335 try:
335 try:
336 used += os.path.getsize(os.path.join(root, f))
336 used += os.path.getsize(os.path.join(root, f))
337 except OSError:
337 except OSError:
338 pass
338 pass
339 value.update({
339 value.update({
340 'percent': 100,
340 'percent': 100,
341 'used': used,
341 'used': used,
342 'total': used,
342 'total': used,
343 'items': items_count
343 'items': items_count
344 })
344 })
345
345
346 except Exception as e:
346 except Exception as e:
347 log.exception('failed to fetch archive cache storage')
347 log.exception('failed to fetch archive cache storage')
348 state = {'message': str(e), 'type': STATE_ERR}
348 state = {'message': str(e), 'type': STATE_ERR}
349
349
350 human_value = value.copy()
350 human_value = value.copy()
351 human_value['used'] = format_byte_size_binary(value['used'])
351 human_value['used'] = format_byte_size_binary(value['used'])
352 human_value['total'] = format_byte_size_binary(value['total'])
352 human_value['total'] = format_byte_size_binary(value['total'])
353 human_value['text'] = "{} ({} items)".format(
353 human_value['text'] = "{} ({} items)".format(
354 human_value['used'], value['items'])
354 human_value['used'], value['items'])
355
355
356 return SysInfoRes(value=value, state=state, human_value=human_value)
356 return SysInfoRes(value=value, state=state, human_value=human_value)
357
357
358
358
359 def storage_gist():
359 def storage_gist():
360 from rhodecode.model.gist import GIST_STORE_LOC
360 from rhodecode.model.gist import GIST_STORE_LOC
361 from rhodecode.model.settings import VcsSettingsModel
361 from rhodecode.model.settings import VcsSettingsModel
362 from rhodecode.lib.utils import safe_str
362 from rhodecode.lib.utils import safe_str
363 from rhodecode.lib.helpers import format_byte_size_binary
363 from rhodecode.lib.helpers import format_byte_size_binary
364 path = safe_str(os.path.join(
364 path = safe_str(os.path.join(
365 VcsSettingsModel().get_repos_location(), GIST_STORE_LOC))
365 VcsSettingsModel().get_repos_location(), GIST_STORE_LOC))
366
366
367 # gist storage
367 # gist storage
368 value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
368 value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
369 state = STATE_OK_DEFAULT
369 state = STATE_OK_DEFAULT
370
370
371 try:
371 try:
372 items_count = 0
372 items_count = 0
373 used = 0
373 used = 0
374 for root, dirs, files in os.walk(path):
374 for root, dirs, files in os.walk(path):
375 if root == path:
375 if root == path:
376 items_count = len(dirs)
376 items_count = len(dirs)
377
377
378 for f in files:
378 for f in files:
379 try:
379 try:
380 used += os.path.getsize(os.path.join(root, f))
380 used += os.path.getsize(os.path.join(root, f))
381 except OSError:
381 except OSError:
382 pass
382 pass
383 value.update({
383 value.update({
384 'percent': 100,
384 'percent': 100,
385 'used': used,
385 'used': used,
386 'total': used,
386 'total': used,
387 'items': items_count
387 'items': items_count
388 })
388 })
389 except Exception as e:
389 except Exception as e:
390 log.exception('failed to fetch gist storage items')
390 log.exception('failed to fetch gist storage items')
391 state = {'message': str(e), 'type': STATE_ERR}
391 state = {'message': str(e), 'type': STATE_ERR}
392
392
393 human_value = value.copy()
393 human_value = value.copy()
394 human_value['used'] = format_byte_size_binary(value['used'])
394 human_value['used'] = format_byte_size_binary(value['used'])
395 human_value['total'] = format_byte_size_binary(value['total'])
395 human_value['total'] = format_byte_size_binary(value['total'])
396 human_value['text'] = "{} ({} items)".format(
396 human_value['text'] = "{} ({} items)".format(
397 human_value['used'], value['items'])
397 human_value['used'], value['items'])
398
398
399 return SysInfoRes(value=value, state=state, human_value=human_value)
399 return SysInfoRes(value=value, state=state, human_value=human_value)
400
400
401
401
402 def storage_temp():
402 def storage_temp():
403 import tempfile
403 import tempfile
404 from rhodecode.lib.helpers import format_byte_size_binary
404 from rhodecode.lib.helpers import format_byte_size_binary
405
405
406 path = tempfile.gettempdir()
406 path = tempfile.gettempdir()
407 value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
407 value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
408 state = STATE_OK_DEFAULT
408 state = STATE_OK_DEFAULT
409
409
410 if not psutil:
410 if not psutil:
411 return SysInfoRes(value=value, state=state)
411 return SysInfoRes(value=value, state=state)
412
412
413 try:
413 try:
414 value.update(dict(psutil.disk_usage(path)._asdict()))
414 value.update(dict(psutil.disk_usage(path)._asdict()))
415 except Exception as e:
415 except Exception as e:
416 log.exception('Failed to fetch temp dir info')
416 log.exception('Failed to fetch temp dir info')
417 state = {'message': str(e), 'type': STATE_ERR}
417 state = {'message': str(e), 'type': STATE_ERR}
418
418
419 human_value = value.copy()
419 human_value = value.copy()
420 human_value['used'] = format_byte_size_binary(value['used'])
420 human_value['used'] = format_byte_size_binary(value['used'])
421 human_value['total'] = format_byte_size_binary(value['total'])
421 human_value['total'] = format_byte_size_binary(value['total'])
422 human_value['text'] = "{}/{}, {}% used".format(
422 human_value['text'] = "{}/{}, {}% used".format(
423 format_byte_size_binary(value['used']),
423 format_byte_size_binary(value['used']),
424 format_byte_size_binary(value['total']),
424 format_byte_size_binary(value['total']),
425 value['percent'])
425 value['percent'])
426
426
427 return SysInfoRes(value=value, state=state, human_value=human_value)
427 return SysInfoRes(value=value, state=state, human_value=human_value)
428
428
429
429
430 def search_info():
430 def search_info():
431 import rhodecode
431 import rhodecode
432 from rhodecode.lib.index import searcher_from_config
432 from rhodecode.lib.index import searcher_from_config
433
433
434 backend = rhodecode.CONFIG.get('search.module', '')
434 backend = rhodecode.CONFIG.get('search.module', '')
435 location = rhodecode.CONFIG.get('search.location', '')
435 location = rhodecode.CONFIG.get('search.location', '')
436
436
437 try:
437 try:
438 searcher = searcher_from_config(rhodecode.CONFIG)
438 searcher = searcher_from_config(rhodecode.CONFIG)
439 searcher = searcher.__class__.__name__
439 searcher = searcher.__class__.__name__
440 except Exception:
440 except Exception:
441 searcher = None
441 searcher = None
442
442
443 value = dict(
443 value = dict(
444 backend=backend, searcher=searcher, location=location, text='')
444 backend=backend, searcher=searcher, location=location, text='')
445 state = STATE_OK_DEFAULT
445 state = STATE_OK_DEFAULT
446
446
447 human_value = value.copy()
447 human_value = value.copy()
448 human_value['text'] = "backend:`{}`".format(human_value['backend'])
448 human_value['text'] = "backend:`{}`".format(human_value['backend'])
449
449
450 return SysInfoRes(value=value, state=state, human_value=human_value)
450 return SysInfoRes(value=value, state=state, human_value=human_value)
451
451
452
452
453 def git_info():
453 def git_info():
454 from rhodecode.lib.vcs.backends import git
454 from rhodecode.lib.vcs.backends import git
455 state = STATE_OK_DEFAULT
455 state = STATE_OK_DEFAULT
456 value = human_value = ''
456 value = human_value = ''
457 try:
457 try:
458 value = git.discover_git_version(raise_on_exc=True)
458 value = git.discover_git_version(raise_on_exc=True)
459 human_value = 'version reported from VCSServer: {}'.format(value)
459 human_value = 'version reported from VCSServer: {}'.format(value)
460 except Exception as e:
460 except Exception as e:
461 state = {'message': str(e), 'type': STATE_ERR}
461 state = {'message': str(e), 'type': STATE_ERR}
462
462
463 return SysInfoRes(value=value, state=state, human_value=human_value)
463 return SysInfoRes(value=value, state=state, human_value=human_value)
464
464
465
465
466 def hg_info():
466 def hg_info():
467 from rhodecode.lib.vcs.backends import hg
467 from rhodecode.lib.vcs.backends import hg
468 state = STATE_OK_DEFAULT
468 state = STATE_OK_DEFAULT
469 value = human_value = ''
469 value = human_value = ''
470 try:
470 try:
471 value = hg.discover_hg_version(raise_on_exc=True)
471 value = hg.discover_hg_version(raise_on_exc=True)
472 human_value = 'version reported from VCSServer: {}'.format(value)
472 human_value = 'version reported from VCSServer: {}'.format(value)
473 except Exception as e:
473 except Exception as e:
474 state = {'message': str(e), 'type': STATE_ERR}
474 state = {'message': str(e), 'type': STATE_ERR}
475 return SysInfoRes(value=value, state=state, human_value=human_value)
475 return SysInfoRes(value=value, state=state, human_value=human_value)
476
476
477
477
478 def svn_info():
478 def svn_info():
479 from rhodecode.lib.vcs.backends import svn
479 from rhodecode.lib.vcs.backends import svn
480 state = STATE_OK_DEFAULT
480 state = STATE_OK_DEFAULT
481 value = human_value = ''
481 value = human_value = ''
482 try:
482 try:
483 value = svn.discover_svn_version(raise_on_exc=True)
483 value = svn.discover_svn_version(raise_on_exc=True)
484 human_value = 'version reported from VCSServer: {}'.format(value)
484 human_value = 'version reported from VCSServer: {}'.format(value)
485 except Exception as e:
485 except Exception as e:
486 state = {'message': str(e), 'type': STATE_ERR}
486 state = {'message': str(e), 'type': STATE_ERR}
487 return SysInfoRes(value=value, state=state, human_value=human_value)
487 return SysInfoRes(value=value, state=state, human_value=human_value)
488
488
489
489
490 def vcs_backends():
490 def vcs_backends():
491 import rhodecode
491 import rhodecode
492 value = rhodecode.CONFIG.get('vcs.backends')
492 value = rhodecode.CONFIG.get('vcs.backends')
493 human_value = 'Enabled backends in order: {}'.format(','.join(value))
493 human_value = 'Enabled backends in order: {}'.format(','.join(value))
494 return SysInfoRes(value=value, human_value=human_value)
494 return SysInfoRes(value=value, human_value=human_value)
495
495
496
496
497 def vcs_server():
497 def vcs_server():
498 import rhodecode
498 import rhodecode
499 from rhodecode.lib.vcs.backends import get_vcsserver_service_data
499 from rhodecode.lib.vcs.backends import get_vcsserver_service_data
500
500
501 server_url = rhodecode.CONFIG.get('vcs.server')
501 server_url = rhodecode.CONFIG.get('vcs.server')
502 enabled = rhodecode.CONFIG.get('vcs.server.enable')
502 enabled = rhodecode.CONFIG.get('vcs.server.enable')
503 protocol = rhodecode.CONFIG.get('vcs.server.protocol') or 'http'
503 protocol = rhodecode.CONFIG.get('vcs.server.protocol') or 'http'
504 state = STATE_OK_DEFAULT
504 state = STATE_OK_DEFAULT
505 version = None
505 version = None
506 workers = 0
506 workers = 0
507
507
508 try:
508 try:
509 data = get_vcsserver_service_data()
509 data = get_vcsserver_service_data()
510 if data and 'version' in data:
510 if data and 'version' in data:
511 version = data['version']
511 version = data['version']
512
512
513 if data and 'config' in data:
513 if data and 'config' in data:
514 conf = data['config']
514 conf = data['config']
515 workers = conf.get('workers', 'NOT AVAILABLE')
515 workers = conf.get('workers', 'NOT AVAILABLE')
516
516
517 connection = 'connected'
517 connection = 'connected'
518 except Exception as e:
518 except Exception as e:
519 connection = 'failed'
519 connection = 'failed'
520 state = {'message': str(e), 'type': STATE_ERR}
520 state = {'message': str(e), 'type': STATE_ERR}
521
521
522 value = dict(
522 value = dict(
523 url=server_url,
523 url=server_url,
524 enabled=enabled,
524 enabled=enabled,
525 protocol=protocol,
525 protocol=protocol,
526 connection=connection,
526 connection=connection,
527 version=version,
527 version=version,
528 text='',
528 text='',
529 )
529 )
530
530
531 human_value = value.copy()
531 human_value = value.copy()
532 human_value['text'] = \
532 human_value['text'] = \
533 '{url}@ver:{ver} via {mode} mode[workers:{workers}], connection:{conn}'.format(
533 '{url}@ver:{ver} via {mode} mode[workers:{workers}], connection:{conn}'.format(
534 url=server_url, ver=version, workers=workers, mode=protocol,
534 url=server_url, ver=version, workers=workers, mode=protocol,
535 conn=connection)
535 conn=connection)
536
536
537 return SysInfoRes(value=value, state=state, human_value=human_value)
537 return SysInfoRes(value=value, state=state, human_value=human_value)
538
538
539
539
540 def rhodecode_app_info():
540 def rhodecode_app_info():
541 import rhodecode
541 import rhodecode
542 edition = rhodecode.CONFIG.get('rhodecode.edition')
542 edition = rhodecode.CONFIG.get('rhodecode.edition')
543
543
544 value = dict(
544 value = dict(
545 rhodecode_version=rhodecode.__version__,
545 rhodecode_version=rhodecode.__version__,
546 rhodecode_lib_path=os.path.abspath(rhodecode.__file__),
546 rhodecode_lib_path=os.path.abspath(rhodecode.__file__),
547 text=''
547 text=''
548 )
548 )
549 human_value = value.copy()
549 human_value = value.copy()
550 human_value['text'] = 'RhodeCode {edition}, version {ver}'.format(
550 human_value['text'] = 'RhodeCode {edition}, version {ver}'.format(
551 edition=edition, ver=value['rhodecode_version']
551 edition=edition, ver=value['rhodecode_version']
552 )
552 )
553 return SysInfoRes(value=value, human_value=human_value)
553 return SysInfoRes(value=value, human_value=human_value)
554
554
555
555
556 def rhodecode_config():
556 def rhodecode_config():
557 import rhodecode
557 import rhodecode
558 import ConfigParser as configparser
559 path = rhodecode.CONFIG.get('__file__')
558 path = rhodecode.CONFIG.get('__file__')
560 rhodecode_ini_safe = rhodecode.CONFIG.copy()
559 rhodecode_ini_safe = rhodecode.CONFIG.copy()
561
560
562 try:
561 try:
563 config = configparser.ConfigParser()
562 config = configparser.ConfigParser()
564 config.read(path)
563 config.read(path)
565 parsed_ini = config
564 parsed_ini = config
566 if parsed_ini.has_section('server:main'):
565 if parsed_ini.has_section('server:main'):
567 parsed_ini = dict(parsed_ini.items('server:main'))
566 parsed_ini = dict(parsed_ini.items('server:main'))
568 except Exception:
567 except Exception:
569 log.exception('Failed to read .ini file for display')
568 log.exception('Failed to read .ini file for display')
570 parsed_ini = {}
569 parsed_ini = {}
571
570
572 cert_path = os.path.join(
571 cert_path = os.path.join(
573 os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(path)))),
572 os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(path)))),
574 '.rccontrol-profile/etc/ca-bundle.crt')
573 '.rccontrol-profile/etc/ca-bundle.crt')
575
574
576 rhodecode_ini_safe['server:main'] = parsed_ini
575 rhodecode_ini_safe['server:main'] = parsed_ini
577
576
578 blacklist = [
577 blacklist = [
579 'rhodecode_license_key',
578 'rhodecode_license_key',
580 'routes.map',
579 'routes.map',
581 'sqlalchemy.db1.url',
580 'sqlalchemy.db1.url',
582 'channelstream.secret',
581 'channelstream.secret',
583 'beaker.session.secret',
582 'beaker.session.secret',
584 'rhodecode.encrypted_values.secret',
583 'rhodecode.encrypted_values.secret',
585 'rhodecode_auth_github_consumer_key',
584 'rhodecode_auth_github_consumer_key',
586 'rhodecode_auth_github_consumer_secret',
585 'rhodecode_auth_github_consumer_secret',
587 'rhodecode_auth_google_consumer_key',
586 'rhodecode_auth_google_consumer_key',
588 'rhodecode_auth_google_consumer_secret',
587 'rhodecode_auth_google_consumer_secret',
589 'rhodecode_auth_bitbucket_consumer_secret',
588 'rhodecode_auth_bitbucket_consumer_secret',
590 'rhodecode_auth_bitbucket_consumer_key',
589 'rhodecode_auth_bitbucket_consumer_key',
591 'rhodecode_auth_twitter_consumer_secret',
590 'rhodecode_auth_twitter_consumer_secret',
592 'rhodecode_auth_twitter_consumer_key',
591 'rhodecode_auth_twitter_consumer_key',
593
592
594 'rhodecode_auth_twitter_secret',
593 'rhodecode_auth_twitter_secret',
595 'rhodecode_auth_github_secret',
594 'rhodecode_auth_github_secret',
596 'rhodecode_auth_google_secret',
595 'rhodecode_auth_google_secret',
597 'rhodecode_auth_bitbucket_secret',
596 'rhodecode_auth_bitbucket_secret',
598
597
599 'appenlight.api_key',
598 'appenlight.api_key',
600 ('app_conf', 'sqlalchemy.db1.url')
599 ('app_conf', 'sqlalchemy.db1.url')
601 ]
600 ]
602 for k in blacklist:
601 for k in blacklist:
603 if isinstance(k, tuple):
602 if isinstance(k, tuple):
604 section, key = k
603 section, key = k
605 if section in rhodecode_ini_safe:
604 if section in rhodecode_ini_safe:
606 rhodecode_ini_safe[section] = '**OBFUSCATED**'
605 rhodecode_ini_safe[section] = '**OBFUSCATED**'
607 else:
606 else:
608 rhodecode_ini_safe.pop(k, None)
607 rhodecode_ini_safe.pop(k, None)
609
608
610 # TODO: maybe put some CONFIG checks here ?
609 # TODO: maybe put some CONFIG checks here ?
611 return SysInfoRes(value={'config': rhodecode_ini_safe,
610 return SysInfoRes(value={'config': rhodecode_ini_safe,
612 'path': path, 'cert_path': cert_path})
611 'path': path, 'cert_path': cert_path})
613
612
614
613
615 def database_info():
614 def database_info():
616 import rhodecode
615 import rhodecode
617 from sqlalchemy.engine import url as engine_url
616 from sqlalchemy.engine import url as engine_url
618 from rhodecode.model.meta import Base as sql_base, Session
617 from rhodecode.model.meta import Base as sql_base, Session
619 from rhodecode.model.db import DbMigrateVersion
618 from rhodecode.model.db import DbMigrateVersion
620
619
621 state = STATE_OK_DEFAULT
620 state = STATE_OK_DEFAULT
622
621
623 db_migrate = DbMigrateVersion.query().filter(
622 db_migrate = DbMigrateVersion.query().filter(
624 DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one()
623 DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one()
625
624
626 db_url_obj = engine_url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url'])
625 db_url_obj = engine_url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url'])
627
626
628 try:
627 try:
629 engine = sql_base.metadata.bind
628 engine = sql_base.metadata.bind
630 db_server_info = engine.dialect._get_server_version_info(
629 db_server_info = engine.dialect._get_server_version_info(
631 Session.connection(bind=engine))
630 Session.connection(bind=engine))
632 db_version = '.'.join(map(str, db_server_info))
631 db_version = '.'.join(map(str, db_server_info))
633 except Exception:
632 except Exception:
634 log.exception('failed to fetch db version')
633 log.exception('failed to fetch db version')
635 db_version = 'UNKNOWN'
634 db_version = 'UNKNOWN'
636
635
637 db_info = dict(
636 db_info = dict(
638 migrate_version=db_migrate.version,
637 migrate_version=db_migrate.version,
639 type=db_url_obj.get_backend_name(),
638 type=db_url_obj.get_backend_name(),
640 version=db_version,
639 version=db_version,
641 url=repr(db_url_obj)
640 url=repr(db_url_obj)
642 )
641 )
643 current_version = db_migrate.version
642 current_version = db_migrate.version
644 expected_version = rhodecode.__dbversion__
643 expected_version = rhodecode.__dbversion__
645 if state['type'] == STATE_OK and current_version != expected_version:
644 if state['type'] == STATE_OK and current_version != expected_version:
646 msg = 'Critical: database schema mismatch, ' \
645 msg = 'Critical: database schema mismatch, ' \
647 'expected version {}, got {}. ' \
646 'expected version {}, got {}. ' \
648 'Please run migrations on your database.'.format(
647 'Please run migrations on your database.'.format(
649 expected_version, current_version)
648 expected_version, current_version)
650 state = {'message': msg, 'type': STATE_ERR}
649 state = {'message': msg, 'type': STATE_ERR}
651
650
652 human_value = db_info.copy()
651 human_value = db_info.copy()
653 human_value['url'] = "{} @ migration version: {}".format(
652 human_value['url'] = "{} @ migration version: {}".format(
654 db_info['url'], db_info['migrate_version'])
653 db_info['url'], db_info['migrate_version'])
655 human_value['version'] = "{} {}".format(db_info['type'], db_info['version'])
654 human_value['version'] = "{} {}".format(db_info['type'], db_info['version'])
656 return SysInfoRes(value=db_info, state=state, human_value=human_value)
655 return SysInfoRes(value=db_info, state=state, human_value=human_value)
657
656
658
657
659 def server_info(environ):
658 def server_info(environ):
660 import rhodecode
659 import rhodecode
661 from rhodecode.lib.base import get_server_ip_addr, get_server_port
660 from rhodecode.lib.base import get_server_ip_addr, get_server_port
662
661
663 value = {
662 value = {
664 'server_ip': '%s:%s' % (
663 'server_ip': '%s:%s' % (
665 get_server_ip_addr(environ, log_errors=False),
664 get_server_ip_addr(environ, log_errors=False),
666 get_server_port(environ)
665 get_server_port(environ)
667 ),
666 ),
668 'server_id': rhodecode.CONFIG.get('instance_id'),
667 'server_id': rhodecode.CONFIG.get('instance_id'),
669 }
668 }
670 return SysInfoRes(value=value)
669 return SysInfoRes(value=value)
671
670
672
671
673 def usage_info():
672 def usage_info():
674 from rhodecode.model.db import User, Repository
673 from rhodecode.model.db import User, Repository
675 value = {
674 value = {
676 'users': User.query().count(),
675 'users': User.query().count(),
677 'users_active': User.query().filter(User.active == True).count(),
676 'users_active': User.query().filter(User.active == True).count(),
678 'repositories': Repository.query().count(),
677 'repositories': Repository.query().count(),
679 'repository_types': {
678 'repository_types': {
680 'hg': Repository.query().filter(
679 'hg': Repository.query().filter(
681 Repository.repo_type == 'hg').count(),
680 Repository.repo_type == 'hg').count(),
682 'git': Repository.query().filter(
681 'git': Repository.query().filter(
683 Repository.repo_type == 'git').count(),
682 Repository.repo_type == 'git').count(),
684 'svn': Repository.query().filter(
683 'svn': Repository.query().filter(
685 Repository.repo_type == 'svn').count(),
684 Repository.repo_type == 'svn').count(),
686 },
685 },
687 }
686 }
688 return SysInfoRes(value=value)
687 return SysInfoRes(value=value)
689
688
690
689
691 def get_system_info(environ):
690 def get_system_info(environ):
692 environ = environ or {}
691 environ = environ or {}
693 return {
692 return {
694 'rhodecode_app': SysInfo(rhodecode_app_info)(),
693 'rhodecode_app': SysInfo(rhodecode_app_info)(),
695 'rhodecode_config': SysInfo(rhodecode_config)(),
694 'rhodecode_config': SysInfo(rhodecode_config)(),
696 'rhodecode_usage': SysInfo(usage_info)(),
695 'rhodecode_usage': SysInfo(usage_info)(),
697 'python': SysInfo(python_info)(),
696 'python': SysInfo(python_info)(),
698 'py_modules': SysInfo(py_modules)(),
697 'py_modules': SysInfo(py_modules)(),
699
698
700 'platform': SysInfo(platform_type)(),
699 'platform': SysInfo(platform_type)(),
701 'server': SysInfo(server_info, environ=environ)(),
700 'server': SysInfo(server_info, environ=environ)(),
702 'database': SysInfo(database_info)(),
701 'database': SysInfo(database_info)(),
703
702
704 'storage': SysInfo(storage)(),
703 'storage': SysInfo(storage)(),
705 'storage_inodes': SysInfo(storage_inodes)(),
704 'storage_inodes': SysInfo(storage_inodes)(),
706 'storage_archive': SysInfo(storage_archives)(),
705 'storage_archive': SysInfo(storage_archives)(),
707 'storage_gist': SysInfo(storage_gist)(),
706 'storage_gist': SysInfo(storage_gist)(),
708 'storage_temp': SysInfo(storage_temp)(),
707 'storage_temp': SysInfo(storage_temp)(),
709
708
710 'search': SysInfo(search_info)(),
709 'search': SysInfo(search_info)(),
711
710
712 'uptime': SysInfo(uptime)(),
711 'uptime': SysInfo(uptime)(),
713 'load': SysInfo(machine_load)(),
712 'load': SysInfo(machine_load)(),
714 'cpu': SysInfo(cpu)(),
713 'cpu': SysInfo(cpu)(),
715 'memory': SysInfo(memory)(),
714 'memory': SysInfo(memory)(),
716
715
717 'vcs_backends': SysInfo(vcs_backends)(),
716 'vcs_backends': SysInfo(vcs_backends)(),
718 'vcs_server': SysInfo(vcs_server)(),
717 'vcs_server': SysInfo(vcs_server)(),
719
718
720 'git': SysInfo(git_info)(),
719 'git': SysInfo(git_info)(),
721 'hg': SysInfo(hg_info)(),
720 'hg': SysInfo(hg_info)(),
722 'svn': SysInfo(svn_info)(),
721 'svn': SysInfo(svn_info)(),
723 }
722 }
@@ -1,545 +1,545 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2014-2017 RhodeCode GmbH
3 # Copyright (C) 2014-2017 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 GIT commit module
22 GIT commit module
23 """
23 """
24
24
25 import re
25 import re
26 import stat
26 import stat
27 from ConfigParser import ConfigParser
28 from itertools import chain
27 from itertools import chain
29 from StringIO import StringIO
28 from StringIO import StringIO
30
29
31 from zope.cachedescriptors.property import Lazy as LazyProperty
30 from zope.cachedescriptors.property import Lazy as LazyProperty
31 from pyramid.compat import configparser
32
32
33 from rhodecode.lib.datelib import utcdate_fromtimestamp
33 from rhodecode.lib.datelib import utcdate_fromtimestamp
34 from rhodecode.lib.utils import safe_unicode, safe_str
34 from rhodecode.lib.utils import safe_unicode, safe_str
35 from rhodecode.lib.utils2 import safe_int
35 from rhodecode.lib.utils2 import safe_int
36 from rhodecode.lib.vcs.conf import settings
36 from rhodecode.lib.vcs.conf import settings
37 from rhodecode.lib.vcs.backends import base
37 from rhodecode.lib.vcs.backends import base
38 from rhodecode.lib.vcs.exceptions import CommitError, NodeDoesNotExistError
38 from rhodecode.lib.vcs.exceptions import CommitError, NodeDoesNotExistError
39 from rhodecode.lib.vcs.nodes import (
39 from rhodecode.lib.vcs.nodes import (
40 FileNode, DirNode, NodeKind, RootNode, SubModuleNode,
40 FileNode, DirNode, NodeKind, RootNode, SubModuleNode,
41 ChangedFileNodesGenerator, AddedFileNodesGenerator,
41 ChangedFileNodesGenerator, AddedFileNodesGenerator,
42 RemovedFileNodesGenerator, LargeFileNode)
42 RemovedFileNodesGenerator, LargeFileNode)
43
43
44
44
45 class GitCommit(base.BaseCommit):
45 class GitCommit(base.BaseCommit):
46 """
46 """
47 Represents state of the repository at single commit id.
47 Represents state of the repository at single commit id.
48 """
48 """
49 _author_property = 'author'
49 _author_property = 'author'
50 _committer_property = 'committer'
50 _committer_property = 'committer'
51 _date_property = 'commit_time'
51 _date_property = 'commit_time'
52 _date_tz_property = 'commit_timezone'
52 _date_tz_property = 'commit_timezone'
53 _message_property = 'message'
53 _message_property = 'message'
54 _parents_property = 'parents'
54 _parents_property = 'parents'
55
55
56 _filter_pre_load = [
56 _filter_pre_load = [
57 # done through a more complex tree walk on parents
57 # done through a more complex tree walk on parents
58 "affected_files",
58 "affected_files",
59 # based on repository cached property
59 # based on repository cached property
60 "branch",
60 "branch",
61 # done through subprocess not remote call
61 # done through subprocess not remote call
62 "children",
62 "children",
63 # done through a more complex tree walk on parents
63 # done through a more complex tree walk on parents
64 "status",
64 "status",
65 # mercurial specific property not supported here
65 # mercurial specific property not supported here
66 "_file_paths",
66 "_file_paths",
67 # mercurial specific property not supported here
67 # mercurial specific property not supported here
68 'obsolete',
68 'obsolete',
69 # mercurial specific property not supported here
69 # mercurial specific property not supported here
70 'phase',
70 'phase',
71 # mercurial specific property not supported here
71 # mercurial specific property not supported here
72 'hidden'
72 'hidden'
73 ]
73 ]
74
74
75 def __init__(self, repository, raw_id, idx, pre_load=None):
75 def __init__(self, repository, raw_id, idx, pre_load=None):
76 self.repository = repository
76 self.repository = repository
77 self._remote = repository._remote
77 self._remote = repository._remote
78 # TODO: johbo: Tweak of raw_id should not be necessary
78 # TODO: johbo: Tweak of raw_id should not be necessary
79 self.raw_id = safe_str(raw_id)
79 self.raw_id = safe_str(raw_id)
80 self.idx = idx
80 self.idx = idx
81
81
82 self._set_bulk_properties(pre_load)
82 self._set_bulk_properties(pre_load)
83
83
84 # caches
84 # caches
85 self._stat_modes = {} # stat info for paths
85 self._stat_modes = {} # stat info for paths
86 self._paths = {} # path processed with parse_tree
86 self._paths = {} # path processed with parse_tree
87 self.nodes = {}
87 self.nodes = {}
88 self._submodules = None
88 self._submodules = None
89
89
90 def _set_bulk_properties(self, pre_load):
90 def _set_bulk_properties(self, pre_load):
91 if not pre_load:
91 if not pre_load:
92 return
92 return
93 pre_load = [entry for entry in pre_load
93 pre_load = [entry for entry in pre_load
94 if entry not in self._filter_pre_load]
94 if entry not in self._filter_pre_load]
95 if not pre_load:
95 if not pre_load:
96 return
96 return
97
97
98 result = self._remote.bulk_request(self.raw_id, pre_load)
98 result = self._remote.bulk_request(self.raw_id, pre_load)
99 for attr, value in result.items():
99 for attr, value in result.items():
100 if attr in ["author", "message"]:
100 if attr in ["author", "message"]:
101 if value:
101 if value:
102 value = safe_unicode(value)
102 value = safe_unicode(value)
103 elif attr == "date":
103 elif attr == "date":
104 value = utcdate_fromtimestamp(*value)
104 value = utcdate_fromtimestamp(*value)
105 elif attr == "parents":
105 elif attr == "parents":
106 value = self._make_commits(value)
106 value = self._make_commits(value)
107 self.__dict__[attr] = value
107 self.__dict__[attr] = value
108
108
109 @LazyProperty
109 @LazyProperty
110 def _commit(self):
110 def _commit(self):
111 return self._remote[self.raw_id]
111 return self._remote[self.raw_id]
112
112
113 @LazyProperty
113 @LazyProperty
114 def _tree_id(self):
114 def _tree_id(self):
115 return self._remote[self._commit['tree']]['id']
115 return self._remote[self._commit['tree']]['id']
116
116
117 @LazyProperty
117 @LazyProperty
118 def id(self):
118 def id(self):
119 return self.raw_id
119 return self.raw_id
120
120
121 @LazyProperty
121 @LazyProperty
122 def short_id(self):
122 def short_id(self):
123 return self.raw_id[:12]
123 return self.raw_id[:12]
124
124
125 @LazyProperty
125 @LazyProperty
126 def message(self):
126 def message(self):
127 return safe_unicode(
127 return safe_unicode(
128 self._remote.commit_attribute(self.id, self._message_property))
128 self._remote.commit_attribute(self.id, self._message_property))
129
129
130 @LazyProperty
130 @LazyProperty
131 def committer(self):
131 def committer(self):
132 return safe_unicode(
132 return safe_unicode(
133 self._remote.commit_attribute(self.id, self._committer_property))
133 self._remote.commit_attribute(self.id, self._committer_property))
134
134
135 @LazyProperty
135 @LazyProperty
136 def author(self):
136 def author(self):
137 return safe_unicode(
137 return safe_unicode(
138 self._remote.commit_attribute(self.id, self._author_property))
138 self._remote.commit_attribute(self.id, self._author_property))
139
139
140 @LazyProperty
140 @LazyProperty
141 def date(self):
141 def date(self):
142 unix_ts, tz = self._remote.get_object_attrs(
142 unix_ts, tz = self._remote.get_object_attrs(
143 self.raw_id, self._date_property, self._date_tz_property)
143 self.raw_id, self._date_property, self._date_tz_property)
144 return utcdate_fromtimestamp(unix_ts, tz)
144 return utcdate_fromtimestamp(unix_ts, tz)
145
145
146 @LazyProperty
146 @LazyProperty
147 def status(self):
147 def status(self):
148 """
148 """
149 Returns modified, added, removed, deleted files for current commit
149 Returns modified, added, removed, deleted files for current commit
150 """
150 """
151 return self.changed, self.added, self.removed
151 return self.changed, self.added, self.removed
152
152
153 @LazyProperty
153 @LazyProperty
154 def tags(self):
154 def tags(self):
155 tags = [safe_unicode(name) for name,
155 tags = [safe_unicode(name) for name,
156 commit_id in self.repository.tags.iteritems()
156 commit_id in self.repository.tags.iteritems()
157 if commit_id == self.raw_id]
157 if commit_id == self.raw_id]
158 return tags
158 return tags
159
159
160 @LazyProperty
160 @LazyProperty
161 def branch(self):
161 def branch(self):
162 for name, commit_id in self.repository.branches.iteritems():
162 for name, commit_id in self.repository.branches.iteritems():
163 if commit_id == self.raw_id:
163 if commit_id == self.raw_id:
164 return safe_unicode(name)
164 return safe_unicode(name)
165 return None
165 return None
166
166
167 def _get_id_for_path(self, path):
167 def _get_id_for_path(self, path):
168 path = safe_str(path)
168 path = safe_str(path)
169 if path in self._paths:
169 if path in self._paths:
170 return self._paths[path]
170 return self._paths[path]
171
171
172 tree_id = self._tree_id
172 tree_id = self._tree_id
173
173
174 path = path.strip('/')
174 path = path.strip('/')
175 if path == '':
175 if path == '':
176 data = [tree_id, "tree"]
176 data = [tree_id, "tree"]
177 self._paths[''] = data
177 self._paths[''] = data
178 return data
178 return data
179
179
180 parts = path.split('/')
180 parts = path.split('/')
181 dirs, name = parts[:-1], parts[-1]
181 dirs, name = parts[:-1], parts[-1]
182 cur_dir = ''
182 cur_dir = ''
183
183
184 # initially extract things from root dir
184 # initially extract things from root dir
185 tree_items = self._remote.tree_items(tree_id)
185 tree_items = self._remote.tree_items(tree_id)
186 self._process_tree_items(tree_items, cur_dir)
186 self._process_tree_items(tree_items, cur_dir)
187
187
188 for dir in dirs:
188 for dir in dirs:
189 if cur_dir:
189 if cur_dir:
190 cur_dir = '/'.join((cur_dir, dir))
190 cur_dir = '/'.join((cur_dir, dir))
191 else:
191 else:
192 cur_dir = dir
192 cur_dir = dir
193 dir_id = None
193 dir_id = None
194 for item, stat_, id_, type_ in tree_items:
194 for item, stat_, id_, type_ in tree_items:
195 if item == dir:
195 if item == dir:
196 dir_id = id_
196 dir_id = id_
197 break
197 break
198 if dir_id:
198 if dir_id:
199 if type_ != "tree":
199 if type_ != "tree":
200 raise CommitError('%s is not a directory' % cur_dir)
200 raise CommitError('%s is not a directory' % cur_dir)
201 # update tree
201 # update tree
202 tree_items = self._remote.tree_items(dir_id)
202 tree_items = self._remote.tree_items(dir_id)
203 else:
203 else:
204 raise CommitError('%s have not been found' % cur_dir)
204 raise CommitError('%s have not been found' % cur_dir)
205
205
206 # cache all items from the given traversed tree
206 # cache all items from the given traversed tree
207 self._process_tree_items(tree_items, cur_dir)
207 self._process_tree_items(tree_items, cur_dir)
208
208
209 if path not in self._paths:
209 if path not in self._paths:
210 raise self.no_node_at_path(path)
210 raise self.no_node_at_path(path)
211
211
212 return self._paths[path]
212 return self._paths[path]
213
213
214 def _process_tree_items(self, items, cur_dir):
214 def _process_tree_items(self, items, cur_dir):
215 for item, stat_, id_, type_ in items:
215 for item, stat_, id_, type_ in items:
216 if cur_dir:
216 if cur_dir:
217 name = '/'.join((cur_dir, item))
217 name = '/'.join((cur_dir, item))
218 else:
218 else:
219 name = item
219 name = item
220 self._paths[name] = [id_, type_]
220 self._paths[name] = [id_, type_]
221 self._stat_modes[name] = stat_
221 self._stat_modes[name] = stat_
222
222
223 def _get_kind(self, path):
223 def _get_kind(self, path):
224 path_id, type_ = self._get_id_for_path(path)
224 path_id, type_ = self._get_id_for_path(path)
225 if type_ == 'blob':
225 if type_ == 'blob':
226 return NodeKind.FILE
226 return NodeKind.FILE
227 elif type_ == 'tree':
227 elif type_ == 'tree':
228 return NodeKind.DIR
228 return NodeKind.DIR
229 elif type == 'link':
229 elif type == 'link':
230 return NodeKind.SUBMODULE
230 return NodeKind.SUBMODULE
231 return None
231 return None
232
232
233 def _get_filectx(self, path):
233 def _get_filectx(self, path):
234 path = self._fix_path(path)
234 path = self._fix_path(path)
235 if self._get_kind(path) != NodeKind.FILE:
235 if self._get_kind(path) != NodeKind.FILE:
236 raise CommitError(
236 raise CommitError(
237 "File does not exist for commit %s at '%s'" %
237 "File does not exist for commit %s at '%s'" %
238 (self.raw_id, path))
238 (self.raw_id, path))
239 return path
239 return path
240
240
241 def _get_file_nodes(self):
241 def _get_file_nodes(self):
242 return chain(*(t[2] for t in self.walk()))
242 return chain(*(t[2] for t in self.walk()))
243
243
244 @LazyProperty
244 @LazyProperty
245 def parents(self):
245 def parents(self):
246 """
246 """
247 Returns list of parent commits.
247 Returns list of parent commits.
248 """
248 """
249 parent_ids = self._remote.commit_attribute(
249 parent_ids = self._remote.commit_attribute(
250 self.id, self._parents_property)
250 self.id, self._parents_property)
251 return self._make_commits(parent_ids)
251 return self._make_commits(parent_ids)
252
252
253 @LazyProperty
253 @LazyProperty
254 def children(self):
254 def children(self):
255 """
255 """
256 Returns list of child commits.
256 Returns list of child commits.
257 """
257 """
258 rev_filter = settings.GIT_REV_FILTER
258 rev_filter = settings.GIT_REV_FILTER
259 output, __ = self.repository.run_git_command(
259 output, __ = self.repository.run_git_command(
260 ['rev-list', '--children'] + rev_filter)
260 ['rev-list', '--children'] + rev_filter)
261
261
262 child_ids = []
262 child_ids = []
263 pat = re.compile(r'^%s' % self.raw_id)
263 pat = re.compile(r'^%s' % self.raw_id)
264 for l in output.splitlines():
264 for l in output.splitlines():
265 if pat.match(l):
265 if pat.match(l):
266 found_ids = l.split(' ')[1:]
266 found_ids = l.split(' ')[1:]
267 child_ids.extend(found_ids)
267 child_ids.extend(found_ids)
268 return self._make_commits(child_ids)
268 return self._make_commits(child_ids)
269
269
270 def _make_commits(self, commit_ids, pre_load=None):
270 def _make_commits(self, commit_ids, pre_load=None):
271 return [
271 return [
272 self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
272 self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
273 for commit_id in commit_ids]
273 for commit_id in commit_ids]
274
274
275 def get_file_mode(self, path):
275 def get_file_mode(self, path):
276 """
276 """
277 Returns stat mode of the file at the given `path`.
277 Returns stat mode of the file at the given `path`.
278 """
278 """
279 path = safe_str(path)
279 path = safe_str(path)
280 # ensure path is traversed
280 # ensure path is traversed
281 self._get_id_for_path(path)
281 self._get_id_for_path(path)
282 return self._stat_modes[path]
282 return self._stat_modes[path]
283
283
284 def is_link(self, path):
284 def is_link(self, path):
285 return stat.S_ISLNK(self.get_file_mode(path))
285 return stat.S_ISLNK(self.get_file_mode(path))
286
286
287 def get_file_content(self, path):
287 def get_file_content(self, path):
288 """
288 """
289 Returns content of the file at given `path`.
289 Returns content of the file at given `path`.
290 """
290 """
291 id_, _ = self._get_id_for_path(path)
291 id_, _ = self._get_id_for_path(path)
292 return self._remote.blob_as_pretty_string(id_)
292 return self._remote.blob_as_pretty_string(id_)
293
293
294 def get_file_size(self, path):
294 def get_file_size(self, path):
295 """
295 """
296 Returns size of the file at given `path`.
296 Returns size of the file at given `path`.
297 """
297 """
298 id_, _ = self._get_id_for_path(path)
298 id_, _ = self._get_id_for_path(path)
299 return self._remote.blob_raw_length(id_)
299 return self._remote.blob_raw_length(id_)
300
300
301 def get_file_history(self, path, limit=None, pre_load=None):
301 def get_file_history(self, path, limit=None, pre_load=None):
302 """
302 """
303 Returns history of file as reversed list of `GitCommit` objects for
303 Returns history of file as reversed list of `GitCommit` objects for
304 which file at given `path` has been modified.
304 which file at given `path` has been modified.
305
305
306 TODO: This function now uses an underlying 'git' command which works
306 TODO: This function now uses an underlying 'git' command which works
307 quickly but ideally we should replace with an algorithm.
307 quickly but ideally we should replace with an algorithm.
308 """
308 """
309 self._get_filectx(path)
309 self._get_filectx(path)
310 f_path = safe_str(path)
310 f_path = safe_str(path)
311
311
312 cmd = ['log']
312 cmd = ['log']
313 if limit:
313 if limit:
314 cmd.extend(['-n', str(safe_int(limit, 0))])
314 cmd.extend(['-n', str(safe_int(limit, 0))])
315 cmd.extend(['--pretty=format: %H', '-s', self.raw_id, '--', f_path])
315 cmd.extend(['--pretty=format: %H', '-s', self.raw_id, '--', f_path])
316
316
317 output, __ = self.repository.run_git_command(cmd)
317 output, __ = self.repository.run_git_command(cmd)
318 commit_ids = re.findall(r'[0-9a-fA-F]{40}', output)
318 commit_ids = re.findall(r'[0-9a-fA-F]{40}', output)
319
319
320 return [
320 return [
321 self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
321 self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
322 for commit_id in commit_ids]
322 for commit_id in commit_ids]
323
323
324 # TODO: unused for now potential replacement for subprocess
324 # TODO: unused for now potential replacement for subprocess
325 def get_file_history_2(self, path, limit=None, pre_load=None):
325 def get_file_history_2(self, path, limit=None, pre_load=None):
326 """
326 """
327 Returns history of file as reversed list of `Commit` objects for
327 Returns history of file as reversed list of `Commit` objects for
328 which file at given `path` has been modified.
328 which file at given `path` has been modified.
329 """
329 """
330 self._get_filectx(path)
330 self._get_filectx(path)
331 f_path = safe_str(path)
331 f_path = safe_str(path)
332
332
333 commit_ids = self._remote.get_file_history(f_path, self.id, limit)
333 commit_ids = self._remote.get_file_history(f_path, self.id, limit)
334
334
335 return [
335 return [
336 self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
336 self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
337 for commit_id in commit_ids]
337 for commit_id in commit_ids]
338
338
339 def get_file_annotate(self, path, pre_load=None):
339 def get_file_annotate(self, path, pre_load=None):
340 """
340 """
341 Returns a generator of four element tuples with
341 Returns a generator of four element tuples with
342 lineno, commit_id, commit lazy loader and line
342 lineno, commit_id, commit lazy loader and line
343
343
344 TODO: This function now uses os underlying 'git' command which is
344 TODO: This function now uses os underlying 'git' command which is
345 generally not good. Should be replaced with algorithm iterating
345 generally not good. Should be replaced with algorithm iterating
346 commits.
346 commits.
347 """
347 """
348 cmd = ['blame', '-l', '--root', '-r', self.raw_id, '--', path]
348 cmd = ['blame', '-l', '--root', '-r', self.raw_id, '--', path]
349 # -l ==> outputs long shas (and we need all 40 characters)
349 # -l ==> outputs long shas (and we need all 40 characters)
350 # --root ==> doesn't put '^' character for bounderies
350 # --root ==> doesn't put '^' character for bounderies
351 # -r commit_id ==> blames for the given commit
351 # -r commit_id ==> blames for the given commit
352 output, __ = self.repository.run_git_command(cmd)
352 output, __ = self.repository.run_git_command(cmd)
353
353
354 for i, blame_line in enumerate(output.split('\n')[:-1]):
354 for i, blame_line in enumerate(output.split('\n')[:-1]):
355 line_no = i + 1
355 line_no = i + 1
356 commit_id, line = re.split(r' ', blame_line, 1)
356 commit_id, line = re.split(r' ', blame_line, 1)
357 yield (
357 yield (
358 line_no, commit_id,
358 line_no, commit_id,
359 lambda: self.repository.get_commit(commit_id=commit_id,
359 lambda: self.repository.get_commit(commit_id=commit_id,
360 pre_load=pre_load),
360 pre_load=pre_load),
361 line)
361 line)
362
362
363 def get_nodes(self, path):
363 def get_nodes(self, path):
364 if self._get_kind(path) != NodeKind.DIR:
364 if self._get_kind(path) != NodeKind.DIR:
365 raise CommitError(
365 raise CommitError(
366 "Directory does not exist for commit %s at "
366 "Directory does not exist for commit %s at "
367 " '%s'" % (self.raw_id, path))
367 " '%s'" % (self.raw_id, path))
368 path = self._fix_path(path)
368 path = self._fix_path(path)
369 id_, _ = self._get_id_for_path(path)
369 id_, _ = self._get_id_for_path(path)
370 tree_id = self._remote[id_]['id']
370 tree_id = self._remote[id_]['id']
371 dirnodes = []
371 dirnodes = []
372 filenodes = []
372 filenodes = []
373 alias = self.repository.alias
373 alias = self.repository.alias
374 for name, stat_, id_, type_ in self._remote.tree_items(tree_id):
374 for name, stat_, id_, type_ in self._remote.tree_items(tree_id):
375 if type_ == 'link':
375 if type_ == 'link':
376 url = self._get_submodule_url('/'.join((path, name)))
376 url = self._get_submodule_url('/'.join((path, name)))
377 dirnodes.append(SubModuleNode(
377 dirnodes.append(SubModuleNode(
378 name, url=url, commit=id_, alias=alias))
378 name, url=url, commit=id_, alias=alias))
379 continue
379 continue
380
380
381 if path != '':
381 if path != '':
382 obj_path = '/'.join((path, name))
382 obj_path = '/'.join((path, name))
383 else:
383 else:
384 obj_path = name
384 obj_path = name
385 if obj_path not in self._stat_modes:
385 if obj_path not in self._stat_modes:
386 self._stat_modes[obj_path] = stat_
386 self._stat_modes[obj_path] = stat_
387
387
388 if type_ == 'tree':
388 if type_ == 'tree':
389 dirnodes.append(DirNode(obj_path, commit=self))
389 dirnodes.append(DirNode(obj_path, commit=self))
390 elif type_ == 'blob':
390 elif type_ == 'blob':
391 filenodes.append(FileNode(obj_path, commit=self, mode=stat_))
391 filenodes.append(FileNode(obj_path, commit=self, mode=stat_))
392 else:
392 else:
393 raise CommitError(
393 raise CommitError(
394 "Requested object should be Tree or Blob, is %s", type_)
394 "Requested object should be Tree or Blob, is %s", type_)
395
395
396 nodes = dirnodes + filenodes
396 nodes = dirnodes + filenodes
397 for node in nodes:
397 for node in nodes:
398 if node.path not in self.nodes:
398 if node.path not in self.nodes:
399 self.nodes[node.path] = node
399 self.nodes[node.path] = node
400 nodes.sort()
400 nodes.sort()
401 return nodes
401 return nodes
402
402
403 def get_node(self, path, pre_load=None):
403 def get_node(self, path, pre_load=None):
404 if isinstance(path, unicode):
404 if isinstance(path, unicode):
405 path = path.encode('utf-8')
405 path = path.encode('utf-8')
406 path = self._fix_path(path)
406 path = self._fix_path(path)
407 if path not in self.nodes:
407 if path not in self.nodes:
408 try:
408 try:
409 id_, type_ = self._get_id_for_path(path)
409 id_, type_ = self._get_id_for_path(path)
410 except CommitError:
410 except CommitError:
411 raise NodeDoesNotExistError(
411 raise NodeDoesNotExistError(
412 "Cannot find one of parents' directories for a given "
412 "Cannot find one of parents' directories for a given "
413 "path: %s" % path)
413 "path: %s" % path)
414
414
415 if type_ == 'link':
415 if type_ == 'link':
416 url = self._get_submodule_url(path)
416 url = self._get_submodule_url(path)
417 node = SubModuleNode(path, url=url, commit=id_,
417 node = SubModuleNode(path, url=url, commit=id_,
418 alias=self.repository.alias)
418 alias=self.repository.alias)
419 elif type_ == 'tree':
419 elif type_ == 'tree':
420 if path == '':
420 if path == '':
421 node = RootNode(commit=self)
421 node = RootNode(commit=self)
422 else:
422 else:
423 node = DirNode(path, commit=self)
423 node = DirNode(path, commit=self)
424 elif type_ == 'blob':
424 elif type_ == 'blob':
425 node = FileNode(path, commit=self, pre_load=pre_load)
425 node = FileNode(path, commit=self, pre_load=pre_load)
426 else:
426 else:
427 raise self.no_node_at_path(path)
427 raise self.no_node_at_path(path)
428
428
429 # cache node
429 # cache node
430 self.nodes[path] = node
430 self.nodes[path] = node
431 return self.nodes[path]
431 return self.nodes[path]
432
432
433 def get_largefile_node(self, path):
433 def get_largefile_node(self, path):
434 id_, _ = self._get_id_for_path(path)
434 id_, _ = self._get_id_for_path(path)
435 pointer_spec = self._remote.is_large_file(id_)
435 pointer_spec = self._remote.is_large_file(id_)
436
436
437 if pointer_spec:
437 if pointer_spec:
438 # content of that file regular FileNode is the hash of largefile
438 # content of that file regular FileNode is the hash of largefile
439 file_id = pointer_spec.get('oid_hash')
439 file_id = pointer_spec.get('oid_hash')
440 if self._remote.in_largefiles_store(file_id):
440 if self._remote.in_largefiles_store(file_id):
441 lf_path = self._remote.store_path(file_id)
441 lf_path = self._remote.store_path(file_id)
442 return LargeFileNode(lf_path, commit=self, org_path=path)
442 return LargeFileNode(lf_path, commit=self, org_path=path)
443
443
444 @LazyProperty
444 @LazyProperty
445 def affected_files(self):
445 def affected_files(self):
446 """
446 """
447 Gets a fast accessible file changes for given commit
447 Gets a fast accessible file changes for given commit
448 """
448 """
449 added, modified, deleted = self._changes_cache
449 added, modified, deleted = self._changes_cache
450 return list(added.union(modified).union(deleted))
450 return list(added.union(modified).union(deleted))
451
451
452 @LazyProperty
452 @LazyProperty
453 def _changes_cache(self):
453 def _changes_cache(self):
454 added = set()
454 added = set()
455 modified = set()
455 modified = set()
456 deleted = set()
456 deleted = set()
457 _r = self._remote
457 _r = self._remote
458
458
459 parents = self.parents
459 parents = self.parents
460 if not self.parents:
460 if not self.parents:
461 parents = [base.EmptyCommit()]
461 parents = [base.EmptyCommit()]
462 for parent in parents:
462 for parent in parents:
463 if isinstance(parent, base.EmptyCommit):
463 if isinstance(parent, base.EmptyCommit):
464 oid = None
464 oid = None
465 else:
465 else:
466 oid = parent.raw_id
466 oid = parent.raw_id
467 changes = _r.tree_changes(oid, self.raw_id)
467 changes = _r.tree_changes(oid, self.raw_id)
468 for (oldpath, newpath), (_, _), (_, _) in changes:
468 for (oldpath, newpath), (_, _), (_, _) in changes:
469 if newpath and oldpath:
469 if newpath and oldpath:
470 modified.add(newpath)
470 modified.add(newpath)
471 elif newpath and not oldpath:
471 elif newpath and not oldpath:
472 added.add(newpath)
472 added.add(newpath)
473 elif not newpath and oldpath:
473 elif not newpath and oldpath:
474 deleted.add(oldpath)
474 deleted.add(oldpath)
475 return added, modified, deleted
475 return added, modified, deleted
476
476
477 def _get_paths_for_status(self, status):
477 def _get_paths_for_status(self, status):
478 """
478 """
479 Returns sorted list of paths for given ``status``.
479 Returns sorted list of paths for given ``status``.
480
480
481 :param status: one of: *added*, *modified* or *deleted*
481 :param status: one of: *added*, *modified* or *deleted*
482 """
482 """
483 added, modified, deleted = self._changes_cache
483 added, modified, deleted = self._changes_cache
484 return sorted({
484 return sorted({
485 'added': list(added),
485 'added': list(added),
486 'modified': list(modified),
486 'modified': list(modified),
487 'deleted': list(deleted)}[status]
487 'deleted': list(deleted)}[status]
488 )
488 )
489
489
490 @LazyProperty
490 @LazyProperty
491 def added(self):
491 def added(self):
492 """
492 """
493 Returns list of added ``FileNode`` objects.
493 Returns list of added ``FileNode`` objects.
494 """
494 """
495 if not self.parents:
495 if not self.parents:
496 return list(self._get_file_nodes())
496 return list(self._get_file_nodes())
497 return AddedFileNodesGenerator(
497 return AddedFileNodesGenerator(
498 [n for n in self._get_paths_for_status('added')], self)
498 [n for n in self._get_paths_for_status('added')], self)
499
499
500 @LazyProperty
500 @LazyProperty
501 def changed(self):
501 def changed(self):
502 """
502 """
503 Returns list of modified ``FileNode`` objects.
503 Returns list of modified ``FileNode`` objects.
504 """
504 """
505 if not self.parents:
505 if not self.parents:
506 return []
506 return []
507 return ChangedFileNodesGenerator(
507 return ChangedFileNodesGenerator(
508 [n for n in self._get_paths_for_status('modified')], self)
508 [n for n in self._get_paths_for_status('modified')], self)
509
509
510 @LazyProperty
510 @LazyProperty
511 def removed(self):
511 def removed(self):
512 """
512 """
513 Returns list of removed ``FileNode`` objects.
513 Returns list of removed ``FileNode`` objects.
514 """
514 """
515 if not self.parents:
515 if not self.parents:
516 return []
516 return []
517 return RemovedFileNodesGenerator(
517 return RemovedFileNodesGenerator(
518 [n for n in self._get_paths_for_status('deleted')], self)
518 [n for n in self._get_paths_for_status('deleted')], self)
519
519
520 def _get_submodule_url(self, submodule_path):
520 def _get_submodule_url(self, submodule_path):
521 git_modules_path = '.gitmodules'
521 git_modules_path = '.gitmodules'
522
522
523 if self._submodules is None:
523 if self._submodules is None:
524 self._submodules = {}
524 self._submodules = {}
525
525
526 try:
526 try:
527 submodules_node = self.get_node(git_modules_path)
527 submodules_node = self.get_node(git_modules_path)
528 except NodeDoesNotExistError:
528 except NodeDoesNotExistError:
529 return None
529 return None
530
530
531 content = submodules_node.content
531 content = submodules_node.content
532
532
533 # ConfigParser fails if there are whitespaces
533 # ConfigParser fails if there are whitespaces
534 content = '\n'.join(l.strip() for l in content.split('\n'))
534 content = '\n'.join(l.strip() for l in content.split('\n'))
535
535
536 parser = ConfigParser()
536 parser = configparser.ConfigParser()
537 parser.readfp(StringIO(content))
537 parser.readfp(StringIO(content))
538
538
539 for section in parser.sections():
539 for section in parser.sections():
540 path = parser.get(section, 'path')
540 path = parser.get(section, 'path')
541 url = parser.get(section, 'url')
541 url = parser.get(section, 'url')
542 if path and url:
542 if path and url:
543 self._submodules[path.strip('/')] = url
543 self._submodules[path.strip('/')] = url
544
544
545 return self._submodules.get(submodule_path.strip('/'))
545 return self._submodules.get(submodule_path.strip('/'))
General Comments 0
You need to be logged in to leave comments. Login now