##// END OF EJS Templates
ssh-wrapper: perf optimizations...
super-admin -
r4947:4d7cf945 default
parent child Browse files
Show More
@@ -1,238 +1,264 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2016-2020 RhodeCode GmbH
3 # Copyright (C) 2016-2020 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import os
21 import os
22 import re
22 import re
23 import logging
23 import logging
24 import datetime
24 import datetime
25 import configparser
25 import configparser
26 from sqlalchemy import Table
26
27
27 from rhodecode.model.db import Session, User, UserSshKeys
28 from rhodecode.lib.utils2 import AttributeDict
28 from rhodecode.model.scm import ScmModel
29 from rhodecode.model.scm import ScmModel
29
30
30 from .hg import MercurialServer
31 from .hg import MercurialServer
31 from .git import GitServer
32 from .git import GitServer
32 from .svn import SubversionServer
33 from .svn import SubversionServer
33 log = logging.getLogger(__name__)
34 log = logging.getLogger(__name__)
34
35
35
36
36 class SshWrapper(object):
37 class SshWrapper(object):
37 hg_cmd_pat = re.compile(r'^hg\s+\-R\s+(\S+)\s+serve\s+\-\-stdio$')
38 hg_cmd_pat = re.compile(r'^hg\s+\-R\s+(\S+)\s+serve\s+\-\-stdio$')
38 git_cmd_pat = re.compile(r'^git-(receive-pack|upload-pack)\s\'[/]?(\S+?)(|\.git)\'$')
39 git_cmd_pat = re.compile(r'^git-(receive-pack|upload-pack)\s\'[/]?(\S+?)(|\.git)\'$')
39 svn_cmd_pat = re.compile(r'^svnserve -t')
40 svn_cmd_pat = re.compile(r'^svnserve -t')
40
41
41 def __init__(self, command, connection_info, mode,
42 def __init__(self, command, connection_info, mode,
42 user, user_id, key_id, shell, ini_path, env):
43 user, user_id, key_id: int, shell, ini_path: str, env):
43 self.command = command
44 self.command = command
44 self.connection_info = connection_info
45 self.connection_info = connection_info
45 self.mode = mode
46 self.mode = mode
46 self.user = user
47 self.username = user
47 self.user_id = user_id
48 self.user_id = user_id
48 self.key_id = key_id
49 self.key_id = key_id
49 self.shell = shell
50 self.shell = shell
50 self.ini_path = ini_path
51 self.ini_path = ini_path
51 self.env = env
52 self.env = env
52
53
53 self.config = self.parse_config(ini_path)
54 self.config = self.parse_config(ini_path)
54 self.server_impl = None
55 self.server_impl = None
55
56
56 def parse_config(self, config_path):
57 def parse_config(self, config_path):
57 parser = configparser.ConfigParser()
58 parser = configparser.ConfigParser()
58 parser.read(config_path)
59 parser.read(config_path)
59 return parser
60 return parser
60
61
61 def update_key_access_time(self, key_id):
62 def update_key_access_time(self, key_id):
62 key = UserSshKeys().query().filter(
63 from rhodecode.model.meta import raw_query_executor, Base
63 UserSshKeys.ssh_key_id == key_id).scalar()
64
64 if key:
65 table = Table('user_ssh_keys', Base.metadata, autoload=False)
65 key.accessed_on = datetime.datetime.utcnow()
66 stmt = (
66 Session().add(key)
67 table.update()
67 Session().commit()
68 .where(table.c.ssh_key_id == key_id)
69 .values(accessed_on=datetime.datetime.utcnow())
70 .returning(table.c.accessed_on, table.c.ssh_key_fingerprint)
71 )
72
73 scalar_res = None
74 with raw_query_executor() as session:
75 result = session.execute(stmt)
76 if result.rowcount:
77 scalar_res = result.first()
78
79 if scalar_res:
80 atime, ssh_key_fingerprint = scalar_res
68 log.debug('Update key id:`%s` fingerprint:`%s` access time',
81 log.debug('Update key id:`%s` fingerprint:`%s` access time',
69 key_id, key.ssh_key_fingerprint)
82 key_id, ssh_key_fingerprint)
83
84 def get_user(self, user_id):
85 user = AttributeDict()
86 # lazy load db imports
87 from rhodecode.model.db import User
88 dbuser = User.get(user_id)
89 if not dbuser:
90 return None
91 user.user_id = dbuser.user_id
92 user.username = dbuser.username
93 user.auth_user = dbuser.AuthUser()
94 return user
70
95
71 def get_connection_info(self):
96 def get_connection_info(self):
72 """
97 """
73 connection_info
98 connection_info
74
99
75 Identifies the client and server ends of the connection.
100 Identifies the client and server ends of the connection.
76 The variable contains four space-separated values: client IP address,
101 The variable contains four space-separated values: client IP address,
77 client port number, server IP address, and server port number.
102 client port number, server IP address, and server port number.
78 """
103 """
79 conn = dict(
104 conn = dict(
80 client_ip=None,
105 client_ip=None,
81 client_port=None,
106 client_port=None,
82 server_ip=None,
107 server_ip=None,
83 server_port=None,
108 server_port=None,
84 )
109 )
85
110
86 info = self.connection_info.split(' ')
111 info = self.connection_info.split(' ')
87 if len(info) == 4:
112 if len(info) == 4:
88 conn['client_ip'] = info[0]
113 conn['client_ip'] = info[0]
89 conn['client_port'] = info[1]
114 conn['client_port'] = info[1]
90 conn['server_ip'] = info[2]
115 conn['server_ip'] = info[2]
91 conn['server_port'] = info[3]
116 conn['server_port'] = info[3]
92
117
93 return conn
118 return conn
94
119
95 def maybe_translate_repo_uid(self, repo_name):
120 def maybe_translate_repo_uid(self, repo_name):
96 _org_name = repo_name
121 _org_name = repo_name
97 if _org_name.startswith('_'):
122 if _org_name.startswith('_'):
98 # remove format of _ID/subrepo
123 # remove format of _ID/subrepo
99 _org_name = _org_name.split('/', 1)[0]
124 _org_name = _org_name.split('/', 1)[0]
100
125
101 if repo_name.startswith('_'):
126 if repo_name.startswith('_'):
102 from rhodecode.model.repo import RepoModel
127 from rhodecode.model.repo import RepoModel
103 org_repo_name = repo_name
128 org_repo_name = repo_name
104 log.debug('translating UID repo %s', org_repo_name)
129 log.debug('translating UID repo %s', org_repo_name)
105 by_id_match = RepoModel().get_repo_by_id(repo_name)
130 by_id_match = RepoModel().get_repo_by_id(repo_name)
106 if by_id_match:
131 if by_id_match:
107 repo_name = by_id_match.repo_name
132 repo_name = by_id_match.repo_name
108 log.debug('translation of UID repo %s got `%s`', org_repo_name, repo_name)
133 log.debug('translation of UID repo %s got `%s`', org_repo_name, repo_name)
109
134
110 return repo_name, _org_name
135 return repo_name, _org_name
111
136
112 def get_repo_details(self, mode):
137 def get_repo_details(self, mode):
113 vcs_type = mode if mode in ['svn', 'hg', 'git'] else None
138 vcs_type = mode if mode in ['svn', 'hg', 'git'] else None
114 repo_name = None
139 repo_name = None
115
140
116 hg_match = self.hg_cmd_pat.match(self.command)
141 hg_match = self.hg_cmd_pat.match(self.command)
117 if hg_match is not None:
142 if hg_match is not None:
118 vcs_type = 'hg'
143 vcs_type = 'hg'
119 repo_id = hg_match.group(1).strip('/')
144 repo_id = hg_match.group(1).strip('/')
120 repo_name, org_name = self.maybe_translate_repo_uid(repo_id)
145 repo_name, org_name = self.maybe_translate_repo_uid(repo_id)
121 return vcs_type, repo_name, mode
146 return vcs_type, repo_name, mode
122
147
123 git_match = self.git_cmd_pat.match(self.command)
148 git_match = self.git_cmd_pat.match(self.command)
124 if git_match is not None:
149 if git_match is not None:
125 mode = git_match.group(1)
150 mode = git_match.group(1)
126 vcs_type = 'git'
151 vcs_type = 'git'
127 repo_id = git_match.group(2).strip('/')
152 repo_id = git_match.group(2).strip('/')
128 repo_name, org_name = self.maybe_translate_repo_uid(repo_id)
153 repo_name, org_name = self.maybe_translate_repo_uid(repo_id)
129 return vcs_type, repo_name, mode
154 return vcs_type, repo_name, mode
130
155
131 svn_match = self.svn_cmd_pat.match(self.command)
156 svn_match = self.svn_cmd_pat.match(self.command)
132 if svn_match is not None:
157 if svn_match is not None:
133 vcs_type = 'svn'
158 vcs_type = 'svn'
134 # Repo name should be extracted from the input stream, we're unable to
159 # Repo name should be extracted from the input stream, we're unable to
135 # extract it at this point in execution
160 # extract it at this point in execution
136 return vcs_type, repo_name, mode
161 return vcs_type, repo_name, mode
137
162
138 return vcs_type, repo_name, mode
163 return vcs_type, repo_name, mode
139
164
140 def serve(self, vcs, repo, mode, user, permissions, branch_permissions):
165 def serve(self, vcs, repo, mode, user, permissions, branch_permissions):
141 store = ScmModel().repos_path
166 store = ScmModel().repos_path
142
167
143 check_branch_perms = False
168 check_branch_perms = False
144 detect_force_push = False
169 detect_force_push = False
145
170
146 if branch_permissions:
171 if branch_permissions:
147 check_branch_perms = True
172 check_branch_perms = True
148 detect_force_push = True
173 detect_force_push = True
149
174
150 log.debug(
175 log.debug(
151 'VCS detected:`%s` mode: `%s` repo_name: %s, branch_permission_checks:%s',
176 'VCS detected:`%s` mode: `%s` repo_name: %s, branch_permission_checks:%s',
152 vcs, mode, repo, check_branch_perms)
177 vcs, mode, repo, check_branch_perms)
153
178
154 # detect if we have to check branch permissions
179 # detect if we have to check branch permissions
155 extras = {
180 extras = {
156 'detect_force_push': detect_force_push,
181 'detect_force_push': detect_force_push,
157 'check_branch_perms': check_branch_perms,
182 'check_branch_perms': check_branch_perms,
158 }
183 }
159
184
160 if vcs == 'hg':
185 if vcs == 'hg':
161 server = MercurialServer(
186 server = MercurialServer(
162 store=store, ini_path=self.ini_path,
187 store=store, ini_path=self.ini_path,
163 repo_name=repo, user=user,
188 repo_name=repo, user=user,
164 user_permissions=permissions, config=self.config, env=self.env)
189 user_permissions=permissions, config=self.config, env=self.env)
165 self.server_impl = server
190 self.server_impl = server
166 return server.run(tunnel_extras=extras)
191 return server.run(tunnel_extras=extras)
167
192
168 elif vcs == 'git':
193 elif vcs == 'git':
169 server = GitServer(
194 server = GitServer(
170 store=store, ini_path=self.ini_path,
195 store=store, ini_path=self.ini_path,
171 repo_name=repo, repo_mode=mode, user=user,
196 repo_name=repo, repo_mode=mode, user=user,
172 user_permissions=permissions, config=self.config, env=self.env)
197 user_permissions=permissions, config=self.config, env=self.env)
173 self.server_impl = server
198 self.server_impl = server
174 return server.run(tunnel_extras=extras)
199 return server.run(tunnel_extras=extras)
175
200
176 elif vcs == 'svn':
201 elif vcs == 'svn':
177 server = SubversionServer(
202 server = SubversionServer(
178 store=store, ini_path=self.ini_path,
203 store=store, ini_path=self.ini_path,
179 repo_name=None, user=user,
204 repo_name=None, user=user,
180 user_permissions=permissions, config=self.config, env=self.env)
205 user_permissions=permissions, config=self.config, env=self.env)
181 self.server_impl = server
206 self.server_impl = server
182 return server.run(tunnel_extras=extras)
207 return server.run(tunnel_extras=extras)
183
208
184 else:
209 else:
185 raise Exception('Unrecognised VCS: {}'.format(vcs))
210 raise Exception('Unrecognised VCS: {}'.format(vcs))
186
211
187 def wrap(self):
212 def wrap(self):
188 mode = self.mode
213 mode = self.mode
189 user = self.user
214 username = self.username
190 user_id = self.user_id
215 user_id = self.user_id
191 key_id = self.key_id
216 key_id = self.key_id
192 shell = self.shell
217 shell = self.shell
193
218
194 scm_detected, scm_repo, scm_mode = self.get_repo_details(mode)
219 scm_detected, scm_repo, scm_mode = self.get_repo_details(mode)
195
220
196 log.debug(
221 log.debug(
197 'Mode: `%s` User: `%s:%s` Shell: `%s` SSH Command: `\"%s\"` '
222 'Mode: `%s` User: `name:%s : id:%s` Shell: `%s` SSH Command: `\"%s\"` '
198 'SCM_DETECTED: `%s` SCM Mode: `%s` SCM Repo: `%s`',
223 'SCM_DETECTED: `%s` SCM Mode: `%s` SCM Repo: `%s`',
199 mode, user, user_id, shell, self.command,
224 mode, username, user_id, shell, self.command,
200 scm_detected, scm_mode, scm_repo)
225 scm_detected, scm_mode, scm_repo)
201
226
227 log.debug('SSH Connection info %s', self.get_connection_info())
228
202 # update last access time for this key
229 # update last access time for this key
203 self.update_key_access_time(key_id)
230 if key_id:
204
231 self.update_key_access_time(key_id)
205 log.debug('SSH Connection info %s', self.get_connection_info())
206
232
207 if shell and self.command is None:
233 if shell and self.command is None:
208 log.info('Dropping to shell, no command given and shell is allowed')
234 log.info('Dropping to shell, no command given and shell is allowed')
209 os.execl('/bin/bash', '-l')
235 os.execl('/bin/bash', '-l')
210 exit_code = 1
236 exit_code = 1
211
237
212 elif scm_detected:
238 elif scm_detected:
213 user = User.get(user_id)
239 user = self.get_user(user_id)
214 if not user:
240 if not user:
215 log.warning('User with id %s not found', user_id)
241 log.warning('User with id %s not found', user_id)
216 exit_code = -1
242 exit_code = -1
217 return exit_code
243 return exit_code
218
244
219 auth_user = user.AuthUser()
245 auth_user = user.auth_user
220 permissions = auth_user.permissions['repositories']
246 permissions = auth_user.permissions['repositories']
221 repo_branch_permissions = auth_user.get_branch_permissions(scm_repo)
247 repo_branch_permissions = auth_user.get_branch_permissions(scm_repo)
222 try:
248 try:
223 exit_code, is_updated = self.serve(
249 exit_code, is_updated = self.serve(
224 scm_detected, scm_repo, scm_mode, user, permissions,
250 scm_detected, scm_repo, scm_mode, user, permissions,
225 repo_branch_permissions)
251 repo_branch_permissions)
226 except Exception:
252 except Exception:
227 log.exception('Error occurred during execution of SshWrapper')
253 log.exception('Error occurred during execution of SshWrapper')
228 exit_code = -1
254 exit_code = -1
229
255
230 elif self.command is None and shell is False:
256 elif self.command is None and shell is False:
231 log.error('No Command given.')
257 log.error('No Command given.')
232 exit_code = -1
258 exit_code = -1
233
259
234 else:
260 else:
235 log.error('Unhandled Command: "%s" Aborting.', self.command)
261 log.error('Unhandled Command: "%s" Aborting.', self.command)
236 exit_code = -1
262 exit_code = -1
237
263
238 return exit_code
264 return exit_code
@@ -1,45 +1,51 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2020 RhodeCode GmbH
3 # Copyright (C) 2010-2020 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 SQLAlchemy Metadata and Session object
22 SQLAlchemy Metadata and Session object
23 """
23 """
24
24
25 from sqlalchemy.orm import declarative_base
25 from sqlalchemy.orm import declarative_base
26 from sqlalchemy.orm import scoped_session, sessionmaker
26 from sqlalchemy.orm import scoped_session, sessionmaker
27
27 from sqlalchemy.orm import Session as SASession
28 from rhodecode.lib import caching_query
28 from rhodecode.lib import caching_query
29
29
30 __all__ = ['Base', 'Session']
30 __all__ = ['Base', 'Session', 'raw_query_executor']
31
31
32 # scoped_session. Apply our custom CachingQuery class to it,
32 # scoped_session. Apply our custom CachingQuery class to it,
33 # using a callable that will associate the dictionary
33 # using a callable that will associate the dictionary
34 # of regions with the Query.
34 # of regions with the Query.
35 # to use cache use this in query
35 # to use cache use this in query
36 # .options(FromCache("sqlalchemy_cache_type", "cachekey"))
36 # .options(FromCache("sqlalchemy_cache_type", "cachekey"))
37 Session = scoped_session(
37 Session = scoped_session(
38 sessionmaker(
38 sessionmaker(
39 query_cls=caching_query.query_callable(),
39 query_cls=caching_query.query_callable(),
40 expire_on_commit=True,
40 expire_on_commit=True,
41 )
41 )
42 )
42 )
43
43
44 # The declarative Base
44 # The declarative Base
45 Base = declarative_base()
45 Base = declarative_base()
46
47
48 def raw_query_executor():
49 engine = Base.metadata.bind
50 session = SASession(engine)
51 return session
General Comments 0
You need to be logged in to leave comments. Login now