##// END OF EJS Templates
rhodecode-vcsserver
RSS

Compare Commits r97:e424cba775fe...r100:2582dee7dd4c

Target:

Source:

Actions:

Compare Commits Swap Comment
Collapse 4 commits
Time Author Commit Description
Martin Bornhold
r97:e424cba775fe
subrepo: Add patch to turn off mercurial subrepo handling.
Martin Bornhold
r98:2fa39df61008
subrepo: Add custom exception to indicate subrepo merge conflict.
Martin Bornhold
r99:e1008af8f033
subrepo: Turn off interactive mode when merging mercurial repo. If there is a merge conflict in the sub repositories mercurial will prompt the used to decide which version to use.
Martin Bornhold
r100:2582dee7dd4c
subrepo: Apply mercurial sub repository patch.
Show file before | Show file after
@@ -1,68 +1,70 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2016 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 """
19 19 Special exception handling over the wire.
20 20
21 21 Since we cannot assume that our client is able to import our exception classes,
22 22 this module provides a "wrapping" mechanism to raise plain exceptions
23 23 which contain an extra attribute `_vcs_kind` to allow a client to distinguish
24 24 different error conditions.
25 25 """
26 26
27 27 import functools
28 28 from pyramid.httpexceptions import HTTPLocked
29 29
30 30
31 31 def _make_exception(kind, *args):
32 32 """
33 33 Prepares a base `Exception` instance to be sent over the wire.
34 34
35 35 To give our caller a hint what this is about, it will attach an attribute
36 36 `_vcs_kind` to the exception.
37 37 """
38 38 exc = Exception(*args)
39 39 exc._vcs_kind = kind
40 40 return exc
41 41
42 42
43 43 AbortException = functools.partial(_make_exception, 'abort')
44 44
45 45 ArchiveException = functools.partial(_make_exception, 'archive')
46 46
47 47 LookupException = functools.partial(_make_exception, 'lookup')
48 48
49 49 VcsException = functools.partial(_make_exception, 'error')
50 50
51 51 RepositoryLockedException = functools.partial(_make_exception, 'repo_locked')
52 52
53 53 RequirementException = functools.partial(_make_exception, 'requirement')
54 54
55 55 UnhandledException = functools.partial(_make_exception, 'unhandled')
56 56
57 57 URLError = functools.partial(_make_exception, 'url_error')
58 58
59 SubrepoMergeException = functools.partial(_make_exception, 'subrepo_merge_error')
60
59 61
60 62 class HTTPRepoLocked(HTTPLocked):
61 63 """
62 64 Subclass of HTTPLocked response that allows to set the title and status
63 65 code via constructor arguments.
64 66 """
65 67 def __init__(self, title, status_code=None, **kwargs):
66 68 self.code = status_code or HTTPLocked.code
67 69 self.title = title
68 70 super(HTTPRepoLocked, self).__init__(**kwargs)
Show file before | Show file after
@@ -1,707 +1,714 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2016 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import io
19 19 import logging
20 20 import stat
21 21 import sys
22 22 import urllib
23 23 import urllib2
24 24
25 25 from hgext import largefiles, rebase
26 26 from hgext.strip import strip as hgext_strip
27 27 from mercurial import commands
28 28 from mercurial import unionrepo
29 29
30 30 from vcsserver import exceptions
31 31 from vcsserver.base import RepoFactory
32 32 from vcsserver.hgcompat import (
33 33 archival, bin, clone, config as hgconfig, diffopts, hex, hg_url,
34 34 httpbasicauthhandler, httpdigestauthhandler, httppeer, localrepository,
35 35 match, memctx, exchange, memfilectx, nullrev, patch, peer, revrange, ui,
36 36 Abort, LookupError, RepoError, RepoLookupError, InterventionRequired,
37 37 RequirementError)
38 38
39 39 log = logging.getLogger(__name__)
40 40
41 41
42 42 def make_ui_from_config(repo_config):
43 43 baseui = ui.ui()
44 44
45 45 # clean the baseui object
46 46 baseui._ocfg = hgconfig.config()
47 47 baseui._ucfg = hgconfig.config()
48 48 baseui._tcfg = hgconfig.config()
49 49
50 50 for section, option, value in repo_config:
51 51 baseui.setconfig(section, option, value)
52 52
53 53 # make our hgweb quiet so it doesn't print output
54 54 baseui.setconfig('ui', 'quiet', 'true')
55 55
56 56 # force mercurial to only use 1 thread, otherwise it may try to set a
57 57 # signal in a non-main thread, thus generating a ValueError.
58 58 baseui.setconfig('worker', 'numcpus', 1)
59 59
60 60 # If there is no config for the largefiles extension, we explicitly disable
61 61 # it here. This overrides settings from repositories hgrc file. Recent
62 62 # mercurial versions enable largefiles in hgrc on clone from largefile
63 63 # repo.
64 64 if not baseui.hasconfig('extensions', 'largefiles'):
65 65 log.debug('Explicitly disable largefiles extension for repo.')
66 66 baseui.setconfig('extensions', 'largefiles', '!')
67 67
68 68 return baseui
69 69
70 70
71 71 def reraise_safe_exceptions(func):
72 72 """Decorator for converting mercurial exceptions to something neutral."""
73 73 def wrapper(*args, **kwargs):
74 74 try:
75 75 return func(*args, **kwargs)
76 76 except (Abort, InterventionRequired):
77 77 raise_from_original(exceptions.AbortException)
78 78 except RepoLookupError:
79 79 raise_from_original(exceptions.LookupException)
80 80 except RequirementError:
81 81 raise_from_original(exceptions.RequirementException)
82 82 except RepoError:
83 83 raise_from_original(exceptions.VcsException)
84 84 except LookupError:
85 85 raise_from_original(exceptions.LookupException)
86 86 except Exception as e:
87 87 if not hasattr(e, '_vcs_kind'):
88 88 log.exception("Unhandled exception in hg remote call")
89 89 raise_from_original(exceptions.UnhandledException)
90 90 raise
91 91 return wrapper
92 92
93 93
94 94 def raise_from_original(new_type):
95 95 """
96 96 Raise a new exception type with original args and traceback.
97 97 """
98 98 _, original, traceback = sys.exc_info()
99 99 try:
100 100 raise new_type(*original.args), None, traceback
101 101 finally:
102 102 del traceback
103 103
104 104
105 105 class MercurialFactory(RepoFactory):
106 106
107 107 def _create_config(self, config, hooks=True):
108 108 if not hooks:
109 109 hooks_to_clean = frozenset((
110 110 'changegroup.repo_size', 'preoutgoing.pre_pull',
111 111 'outgoing.pull_logger', 'prechangegroup.pre_push'))
112 112 new_config = []
113 113 for section, option, value in config:
114 114 if section == 'hooks' and option in hooks_to_clean:
115 115 continue
116 116 new_config.append((section, option, value))
117 117 config = new_config
118 118
119 119 baseui = make_ui_from_config(config)
120 120 return baseui
121 121
122 122 def _create_repo(self, wire, create):
123 123 baseui = self._create_config(wire["config"])
124 124 return localrepository(baseui, wire["path"], create)
125 125
126 126
127 127 class HgRemote(object):
128 128
129 129 def __init__(self, factory):
130 130 self._factory = factory
131 131
132 132 self._bulk_methods = {
133 133 "affected_files": self.ctx_files,
134 134 "author": self.ctx_user,
135 135 "branch": self.ctx_branch,
136 136 "children": self.ctx_children,
137 137 "date": self.ctx_date,
138 138 "message": self.ctx_description,
139 139 "parents": self.ctx_parents,
140 140 "status": self.ctx_status,
141 141 "_file_paths": self.ctx_list,
142 142 }
143 143
144 144 @reraise_safe_exceptions
145 145 def archive_repo(self, archive_path, mtime, file_info, kind):
146 146 if kind == "tgz":
147 147 archiver = archival.tarit(archive_path, mtime, "gz")
148 148 elif kind == "tbz2":
149 149 archiver = archival.tarit(archive_path, mtime, "bz2")
150 150 elif kind == 'zip':
151 151 archiver = archival.zipit(archive_path, mtime)
152 152 else:
153 153 raise exceptions.ArchiveException(
154 154 'Remote does not support: "%s".' % kind)
155 155
156 156 for f_path, f_mode, f_is_link, f_content in file_info:
157 157 archiver.addfile(f_path, f_mode, f_is_link, f_content)
158 158 archiver.done()
159 159
160 160 @reraise_safe_exceptions
161 161 def bookmarks(self, wire):
162 162 repo = self._factory.repo(wire)
163 163 return dict(repo._bookmarks)
164 164
165 165 @reraise_safe_exceptions
166 166 def branches(self, wire, normal, closed):
167 167 repo = self._factory.repo(wire)
168 168 iter_branches = repo.branchmap().iterbranches()
169 169 bt = {}
170 170 for branch_name, _heads, tip, is_closed in iter_branches:
171 171 if normal and not is_closed:
172 172 bt[branch_name] = tip
173 173 if closed and is_closed:
174 174 bt[branch_name] = tip
175 175
176 176 return bt
177 177
178 178 @reraise_safe_exceptions
179 179 def bulk_request(self, wire, rev, pre_load):
180 180 result = {}
181 181 for attr in pre_load:
182 182 try:
183 183 method = self._bulk_methods[attr]
184 184 result[attr] = method(wire, rev)
185 185 except KeyError:
186 186 raise exceptions.VcsException(
187 187 'Unknown bulk attribute: "%s"' % attr)
188 188 return result
189 189
190 190 @reraise_safe_exceptions
191 191 def clone(self, wire, source, dest, update_after_clone=False, hooks=True):
192 192 baseui = self._factory._create_config(wire["config"], hooks=hooks)
193 193 clone(baseui, source, dest, noupdate=not update_after_clone)
194 194
195 195 @reraise_safe_exceptions
196 196 def commitctx(
197 197 self, wire, message, parents, commit_time, commit_timezone,
198 198 user, files, extra, removed, updated):
199 199
200 200 def _filectxfn(_repo, memctx, path):
201 201 """
202 202 Marks given path as added/changed/removed in a given _repo. This is
203 203 for internal mercurial commit function.
204 204 """
205 205
206 206 # check if this path is removed
207 207 if path in removed:
208 208 # returning None is a way to mark node for removal
209 209 return None
210 210
211 211 # check if this path is added
212 212 for node in updated:
213 213 if node['path'] == path:
214 214 return memfilectx(
215 215 _repo,
216 216 path=node['path'],
217 217 data=node['content'],
218 218 islink=False,
219 219 isexec=bool(node['mode'] & stat.S_IXUSR),
220 220 copied=False,
221 221 memctx=memctx)
222 222
223 223 raise exceptions.AbortException(
224 224 "Given path haven't been marked as added, "
225 225 "changed or removed (%s)" % path)
226 226
227 227 repo = self._factory.repo(wire)
228 228
229 229 commit_ctx = memctx(
230 230 repo=repo,
231 231 parents=parents,
232 232 text=message,
233 233 files=files,
234 234 filectxfn=_filectxfn,
235 235 user=user,
236 236 date=(commit_time, commit_timezone),
237 237 extra=extra)
238 238
239 239 n = repo.commitctx(commit_ctx)
240 240 new_id = hex(n)
241 241
242 242 return new_id
243 243
244 244 @reraise_safe_exceptions
245 245 def ctx_branch(self, wire, revision):
246 246 repo = self._factory.repo(wire)
247 247 ctx = repo[revision]
248 248 return ctx.branch()
249 249
250 250 @reraise_safe_exceptions
251 251 def ctx_children(self, wire, revision):
252 252 repo = self._factory.repo(wire)
253 253 ctx = repo[revision]
254 254 return [child.rev() for child in ctx.children()]
255 255
256 256 @reraise_safe_exceptions
257 257 def ctx_date(self, wire, revision):
258 258 repo = self._factory.repo(wire)
259 259 ctx = repo[revision]
260 260 return ctx.date()
261 261
262 262 @reraise_safe_exceptions
263 263 def ctx_description(self, wire, revision):
264 264 repo = self._factory.repo(wire)
265 265 ctx = repo[revision]
266 266 return ctx.description()
267 267
268 268 @reraise_safe_exceptions
269 269 def ctx_diff(
270 270 self, wire, revision, git=True, ignore_whitespace=True, context=3):
271 271 repo = self._factory.repo(wire)
272 272 ctx = repo[revision]
273 273 result = ctx.diff(
274 274 git=git, ignore_whitespace=ignore_whitespace, context=context)
275 275 return list(result)
276 276
277 277 @reraise_safe_exceptions
278 278 def ctx_files(self, wire, revision):
279 279 repo = self._factory.repo(wire)
280 280 ctx = repo[revision]
281 281 return ctx.files()
282 282
283 283 @reraise_safe_exceptions
284 284 def ctx_list(self, path, revision):
285 285 repo = self._factory.repo(path)
286 286 ctx = repo[revision]
287 287 return list(ctx)
288 288
289 289 @reraise_safe_exceptions
290 290 def ctx_parents(self, wire, revision):
291 291 repo = self._factory.repo(wire)
292 292 ctx = repo[revision]
293 293 return [parent.rev() for parent in ctx.parents()]
294 294
295 295 @reraise_safe_exceptions
296 296 def ctx_substate(self, wire, revision):
297 297 repo = self._factory.repo(wire)
298 298 ctx = repo[revision]
299 299 return ctx.substate
300 300
301 301 @reraise_safe_exceptions
302 302 def ctx_status(self, wire, revision):
303 303 repo = self._factory.repo(wire)
304 304 ctx = repo[revision]
305 305 status = repo[ctx.p1().node()].status(other=ctx.node())
306 306 # object of status (odd, custom named tuple in mercurial) is not
307 307 # correctly serializable via Pyro, we make it a list, as the underling
308 308 # API expects this to be a list
309 309 return list(status)
310 310
311 311 @reraise_safe_exceptions
312 312 def ctx_user(self, wire, revision):
313 313 repo = self._factory.repo(wire)
314 314 ctx = repo[revision]
315 315 return ctx.user()
316 316
317 317 @reraise_safe_exceptions
318 318 def check_url(self, url, config):
319 319 log.info("Checking URL for remote cloning/import: %s", url)
320 320 _proto = None
321 321 if '+' in url[:url.find('://')]:
322 322 _proto = url[0:url.find('+')]
323 323 url = url[url.find('+') + 1:]
324 324 handlers = []
325 325 url_obj = hg_url(url)
326 326 test_uri, authinfo = url_obj.authinfo()
327 327 url_obj.passwd = '*****'
328 328 cleaned_uri = str(url_obj)
329 329
330 330 if authinfo:
331 331 # create a password manager
332 332 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
333 333 passmgr.add_password(*authinfo)
334 334
335 335 handlers.extend((httpbasicauthhandler(passmgr),
336 336 httpdigestauthhandler(passmgr)))
337 337
338 338 o = urllib2.build_opener(*handlers)
339 339 o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
340 340 ('Accept', 'application/mercurial-0.1')]
341 341
342 342 q = {"cmd": 'between'}
343 343 q.update({'pairs': "%s-%s" % ('0' * 40, '0' * 40)})
344 344 qs = '?%s' % urllib.urlencode(q)
345 345 cu = "%s%s" % (test_uri, qs)
346 346 req = urllib2.Request(cu, None, {})
347 347
348 348 try:
349 349 log.debug("Trying to open URL %s", url)
350 350 resp = o.open(req)
351 351 if resp.code != 200:
352 352 raise exceptions.URLError('Return Code is not 200')
353 353 except Exception as e:
354 354 log.warning("URL cannot be opened: %s", url, exc_info=True)
355 355 # means it cannot be cloned
356 356 raise exceptions.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
357 357
358 358 # now check if it's a proper hg repo, but don't do it for svn
359 359 try:
360 360 if _proto == 'svn':
361 361 pass
362 362 else:
363 363 # check for pure hg repos
364 364 log.debug(
365 365 "Verifying if URL is a Mercurial repository: %s", url)
366 366 httppeer(make_ui_from_config(config), url).lookup('tip')
367 367 except Exception as e:
368 368 log.warning("URL is not a valid Mercurial repository: %s", url)
369 369 raise exceptions.URLError(
370 370 "url [%s] does not look like an hg repo org_exc: %s"
371 371 % (cleaned_uri, e))
372 372
373 373 log.info("URL is a valid Mercurial repository: %s", url)
374 374 return True
375 375
376 376 @reraise_safe_exceptions
377 377 def diff(
378 378 self, wire, rev1, rev2, file_filter, opt_git, opt_ignorews,
379 379 context):
380 380 repo = self._factory.repo(wire)
381 381
382 382 if file_filter:
383 383 filter = match(file_filter[0], '', [file_filter[1]])
384 384 else:
385 385 filter = file_filter
386 386 opts = diffopts(git=opt_git, ignorews=opt_ignorews, context=context)
387 387
388 388 try:
389 389 return "".join(patch.diff(
390 390 repo, node1=rev1, node2=rev2, match=filter, opts=opts))
391 391 except RepoLookupError:
392 392 raise exceptions.LookupException()
393 393
394 394 @reraise_safe_exceptions
395 395 def file_history(self, wire, revision, path, limit):
396 396 repo = self._factory.repo(wire)
397 397
398 398 ctx = repo[revision]
399 399 fctx = ctx.filectx(path)
400 400
401 401 def history_iter():
402 402 limit_rev = fctx.rev()
403 403 for obj in reversed(list(fctx.filelog())):
404 404 obj = fctx.filectx(obj)
405 405 if limit_rev >= obj.rev():
406 406 yield obj
407 407
408 408 history = []
409 409 for cnt, obj in enumerate(history_iter()):
410 410 if limit and cnt >= limit:
411 411 break
412 412 history.append(hex(obj.node()))
413 413
414 414 return [x for x in history]
415 415
416 416 @reraise_safe_exceptions
417 417 def file_history_untill(self, wire, revision, path, limit):
418 418 repo = self._factory.repo(wire)
419 419 ctx = repo[revision]
420 420 fctx = ctx.filectx(path)
421 421
422 422 file_log = list(fctx.filelog())
423 423 if limit:
424 424 # Limit to the last n items
425 425 file_log = file_log[-limit:]
426 426
427 427 return [hex(fctx.filectx(cs).node()) for cs in reversed(file_log)]
428 428
429 429 @reraise_safe_exceptions
430 430 def fctx_annotate(self, wire, revision, path):
431 431 repo = self._factory.repo(wire)
432 432 ctx = repo[revision]
433 433 fctx = ctx.filectx(path)
434 434
435 435 result = []
436 436 for i, annotate_data in enumerate(fctx.annotate()):
437 437 ln_no = i + 1
438 438 sha = hex(annotate_data[0].node())
439 439 result.append((ln_no, sha, annotate_data[1]))
440 440 return result
441 441
442 442 @reraise_safe_exceptions
443 443 def fctx_data(self, wire, revision, path):
444 444 repo = self._factory.repo(wire)
445 445 ctx = repo[revision]
446 446 fctx = ctx.filectx(path)
447 447 return fctx.data()
448 448
449 449 @reraise_safe_exceptions
450 450 def fctx_flags(self, wire, revision, path):
451 451 repo = self._factory.repo(wire)
452 452 ctx = repo[revision]
453 453 fctx = ctx.filectx(path)
454 454 return fctx.flags()
455 455
456 456 @reraise_safe_exceptions
457 457 def fctx_size(self, wire, revision, path):
458 458 repo = self._factory.repo(wire)
459 459 ctx = repo[revision]
460 460 fctx = ctx.filectx(path)
461 461 return fctx.size()
462 462
463 463 @reraise_safe_exceptions
464 464 def get_all_commit_ids(self, wire, name):
465 465 repo = self._factory.repo(wire)
466 466 revs = repo.filtered(name).changelog.index
467 467 return map(lambda x: hex(x[7]), revs)[:-1]
468 468
469 469 @reraise_safe_exceptions
470 470 def get_config_value(self, wire, section, name, untrusted=False):
471 471 repo = self._factory.repo(wire)
472 472 return repo.ui.config(section, name, untrusted=untrusted)
473 473
474 474 @reraise_safe_exceptions
475 475 def get_config_bool(self, wire, section, name, untrusted=False):
476 476 repo = self._factory.repo(wire)
477 477 return repo.ui.configbool(section, name, untrusted=untrusted)
478 478
479 479 @reraise_safe_exceptions
480 480 def get_config_list(self, wire, section, name, untrusted=False):
481 481 repo = self._factory.repo(wire)
482 482 return repo.ui.configlist(section, name, untrusted=untrusted)
483 483
484 484 @reraise_safe_exceptions
485 485 def is_large_file(self, wire, path):
486 486 return largefiles.lfutil.isstandin(path)
487 487
488 488 @reraise_safe_exceptions
489 489 def in_store(self, wire, sha):
490 490 repo = self._factory.repo(wire)
491 491 return largefiles.lfutil.instore(repo, sha)
492 492
493 493 @reraise_safe_exceptions
494 494 def in_user_cache(self, wire, sha):
495 495 repo = self._factory.repo(wire)
496 496 return largefiles.lfutil.inusercache(repo.ui, sha)
497 497
498 498 @reraise_safe_exceptions
499 499 def store_path(self, wire, sha):
500 500 repo = self._factory.repo(wire)
501 501 return largefiles.lfutil.storepath(repo, sha)
502 502
503 503 @reraise_safe_exceptions
504 504 def link(self, wire, sha, path):
505 505 repo = self._factory.repo(wire)
506 506 largefiles.lfutil.link(
507 507 largefiles.lfutil.usercachepath(repo.ui, sha), path)
508 508
509 509 @reraise_safe_exceptions
510 510 def localrepository(self, wire, create=False):
511 511 self._factory.repo(wire, create=create)
512 512
513 513 @reraise_safe_exceptions
514 514 def lookup(self, wire, revision, both):
515 515 # TODO Paris: Ugly hack to "deserialize" long for msgpack
516 516 if isinstance(revision, float):
517 517 revision = long(revision)
518 518 repo = self._factory.repo(wire)
519 519 try:
520 520 ctx = repo[revision]
521 521 except RepoLookupError:
522 522 raise exceptions.LookupException(revision)
523 523 except LookupError as e:
524 524 raise exceptions.LookupException(e.name)
525 525
526 526 if not both:
527 527 return ctx.hex()
528 528
529 529 ctx = repo[ctx.hex()]
530 530 return ctx.hex(), ctx.rev()
531 531
532 532 @reraise_safe_exceptions
533 533 def pull(self, wire, url, commit_ids=None):
534 534 repo = self._factory.repo(wire)
535 535 remote = peer(repo, {}, url)
536 536 if commit_ids:
537 537 commit_ids = [bin(commit_id) for commit_id in commit_ids]
538 538
539 539 return exchange.pull(
540 540 repo, remote, heads=commit_ids, force=None).cgresult
541 541
542 542 @reraise_safe_exceptions
543 543 def revision(self, wire, rev):
544 544 repo = self._factory.repo(wire)
545 545 ctx = repo[rev]
546 546 return ctx.rev()
547 547
548 548 @reraise_safe_exceptions
549 549 def rev_range(self, wire, filter):
550 550 repo = self._factory.repo(wire)
551 551 revisions = [rev for rev in revrange(repo, filter)]
552 552 return revisions
553 553
554 554 @reraise_safe_exceptions
555 555 def rev_range_hash(self, wire, node):
556 556 repo = self._factory.repo(wire)
557 557
558 558 def get_revs(repo, rev_opt):
559 559 if rev_opt:
560 560 revs = revrange(repo, rev_opt)
561 561 if len(revs) == 0:
562 562 return (nullrev, nullrev)
563 563 return max(revs), min(revs)
564 564 else:
565 565 return len(repo) - 1, 0
566 566
567 567 stop, start = get_revs(repo, [node + ':'])
568 568 revs = [hex(repo[r].node()) for r in xrange(start, stop + 1)]
569 569 return revs
570 570
571 571 @reraise_safe_exceptions
572 572 def revs_from_revspec(self, wire, rev_spec, *args, **kwargs):
573 573 other_path = kwargs.pop('other_path', None)
574 574
575 575 # case when we want to compare two independent repositories
576 576 if other_path and other_path != wire["path"]:
577 577 baseui = self._factory._create_config(wire["config"])
578 578 repo = unionrepo.unionrepository(baseui, other_path, wire["path"])
579 579 else:
580 580 repo = self._factory.repo(wire)
581 581 return list(repo.revs(rev_spec, *args))
582 582
583 583 @reraise_safe_exceptions
584 584 def strip(self, wire, revision, update, backup):
585 585 repo = self._factory.repo(wire)
586 586 ctx = repo[revision]
587 587 hgext_strip(
588 588 repo.baseui, repo, ctx.node(), update=update, backup=backup)
589 589
590 590 @reraise_safe_exceptions
591 591 def tag(self, wire, name, revision, message, local, user,
592 592 tag_time, tag_timezone):
593 593 repo = self._factory.repo(wire)
594 594 ctx = repo[revision]
595 595 node = ctx.node()
596 596
597 597 date = (tag_time, tag_timezone)
598 598 try:
599 599 repo.tag(name, node, message, local, user, date)
600 600 except Abort:
601 601 log.exception("Tag operation aborted")
602 602 raise exceptions.AbortException()
603 603
604 604 @reraise_safe_exceptions
605 605 def tags(self, wire):
606 606 repo = self._factory.repo(wire)
607 607 return repo.tags()
608 608
609 609 @reraise_safe_exceptions
610 610 def update(self, wire, node=None, clean=False):
611 611 repo = self._factory.repo(wire)
612 612 baseui = self._factory._create_config(wire['config'])
613 613 commands.update(baseui, repo, node=node, clean=clean)
614 614
615 615 @reraise_safe_exceptions
616 616 def identify(self, wire):
617 617 repo = self._factory.repo(wire)
618 618 baseui = self._factory._create_config(wire['config'])
619 619 output = io.BytesIO()
620 620 baseui.write = output.write
621 621 # This is required to get a full node id
622 622 baseui.debugflag = True
623 623 commands.identify(baseui, repo, id=True)
624 624
625 625 return output.getvalue()
626 626
627 627 @reraise_safe_exceptions
628 628 def pull_cmd(self, wire, source, bookmark=None, branch=None, revision=None,
629 629 hooks=True):
630 630 repo = self._factory.repo(wire)
631 631 baseui = self._factory._create_config(wire['config'], hooks=hooks)
632 632
633 633 # Mercurial internally has a lot of logic that checks ONLY if
634 634 # option is defined, we just pass those if they are defined then
635 635 opts = {}
636 636 if bookmark:
637 637 opts['bookmark'] = bookmark
638 638 if branch:
639 639 opts['branch'] = branch
640 640 if revision:
641 641 opts['rev'] = revision
642 642
643 643 commands.pull(baseui, repo, source, **opts)
644 644
645 645 @reraise_safe_exceptions
646 646 def heads(self, wire, branch=None):
647 647 repo = self._factory.repo(wire)
648 648 baseui = self._factory._create_config(wire['config'])
649 649 output = io.BytesIO()
650 650
651 651 def write(data, **unused_kwargs):
652 652 output.write(data)
653 653
654 654 baseui.write = write
655 655 if branch:
656 656 args = [branch]
657 657 else:
658 658 args = []
659 659 commands.heads(baseui, repo, template='{node} ', *args)
660 660
661 661 return output.getvalue()
662 662
663 663 @reraise_safe_exceptions
664 664 def ancestor(self, wire, revision1, revision2):
665 665 repo = self._factory.repo(wire)
666 666 baseui = self._factory._create_config(wire['config'])
667 667 output = io.BytesIO()
668 668 baseui.write = output.write
669 669 commands.debugancestor(baseui, repo, revision1, revision2)
670 670
671 671 return output.getvalue()
672 672
673 673 @reraise_safe_exceptions
674 674 def push(self, wire, revisions, dest_path, hooks=True,
675 675 push_branches=False):
676 676 repo = self._factory.repo(wire)
677 677 baseui = self._factory._create_config(wire['config'], hooks=hooks)
678 678 commands.push(baseui, repo, dest=dest_path, rev=revisions,
679 679 new_branch=push_branches)
680 680
681 681 @reraise_safe_exceptions
682 682 def merge(self, wire, revision):
683 683 repo = self._factory.repo(wire)
684 684 baseui = self._factory._create_config(wire['config'])
685 685 repo.ui.setconfig('ui', 'merge', 'internal:dump')
686
687 # In case of sub repositories are used mercurial prompts the user in
688 # case of merge conflicts or different sub repository sources. By
689 # setting the interactive flag to `False` mercurial doesn't prompt the
690 # used but instead uses a default value.
691 repo.ui.setconfig('ui', 'interactive', False)
692
686 693 commands.merge(baseui, repo, rev=revision)
687 694
688 695 @reraise_safe_exceptions
689 696 def commit(self, wire, message, username):
690 697 repo = self._factory.repo(wire)
691 698 baseui = self._factory._create_config(wire['config'])
692 699 repo.ui.setconfig('ui', 'username', username)
693 700 commands.commit(baseui, repo, message=message)
694 701
695 702 @reraise_safe_exceptions
696 703 def rebase(self, wire, source=None, dest=None, abort=False):
697 704 repo = self._factory.repo(wire)
698 705 baseui = self._factory._create_config(wire['config'])
699 706 repo.ui.setconfig('ui', 'merge', 'internal:dump')
700 707 rebase.rebase(
701 708 baseui, repo, base=source, dest=dest, abort=abort, keep=not abort)
702 709
703 710 @reraise_safe_exceptions
704 711 def bookmark(self, wire, bookmark, revision=None):
705 712 repo = self._factory.repo(wire)
706 713 baseui = self._factory._create_config(wire['config'])
707 714 commands.bookmark(baseui, repo, bookmark, rev=revision, force=True)
Show file before | Show file after
@@ -1,61 +1,62 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2016 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 """
19 19 Mercurial libs compatibility
20 20 """
21 21
22 22 import mercurial
23 23 import mercurial.demandimport
24 24 # patch demandimport, due to bug in mercurial when it always triggers
25 25 # demandimport.enable()
26 26 mercurial.demandimport.enable = lambda *args, **kwargs: 1
27 27
28 28 from mercurial import ui
29 29 from mercurial import patch
30 30 from mercurial import config
31 31 from mercurial import extensions
32 32 from mercurial import scmutil
33 33 from mercurial import archival
34 34 from mercurial import discovery
35 35 from mercurial import unionrepo
36 36 from mercurial import localrepo
37 37 from mercurial import merge as hg_merge
38 from mercurial import subrepo
38 39
39 40 from mercurial.commands import clone, nullid, pull
40 41 from mercurial.context import memctx, memfilectx
41 42 from mercurial.error import (
42 43 LookupError, RepoError, RepoLookupError, Abort, InterventionRequired,
43 44 RequirementError)
44 45 from mercurial.hgweb import hgweb_mod
45 46 from mercurial.localrepo import localrepository
46 47 from mercurial.match import match
47 48 from mercurial.mdiff import diffopts
48 49 from mercurial.node import bin, hex
49 50 from mercurial.encoding import tolocal
50 51 from mercurial.discovery import findcommonoutgoing
51 52 from mercurial.hg import peer
52 53 from mercurial.httppeer import httppeer
53 54 from mercurial.util import url as hg_url
54 55 from mercurial.scmutil import revrange
55 56 from mercurial.node import nullrev
56 57 from mercurial import exchange
57 58 from hgext import largefiles
58 59
59 60 # those authnadlers are patched for python 2.6.5 bug an
60 61 # infinit looping when given invalid resources
61 62 from mercurial.url import httpbasicauthhandler, httpdigestauthhandler
Show file before | Show file after
@@ -1,60 +1,134 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2016 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 """
19 19 Adjustments to Mercurial
20 20
21 21 Intentionally kept separate from `hgcompat` and `hg`, so that these patches can
22 22 be applied without having to import the whole Mercurial machinery.
23 23
24 24 Imports are function local, so that just importing this module does not cause
25 25 side-effects other than these functions being defined.
26 26 """
27 27
28 28 import logging
29 29
30 30
31 31 def patch_largefiles_capabilities():
32 32 """
33 33 Patches the capabilities function in the largefiles extension.
34 34 """
35 35 from vcsserver import hgcompat
36 36 lfproto = hgcompat.largefiles.proto
37 37 wrapper = _dynamic_capabilities_wrapper(
38 38 lfproto, hgcompat.extensions.extensions)
39 39 lfproto.capabilities = wrapper
40 40
41 41
42 42 def _dynamic_capabilities_wrapper(lfproto, extensions):
43 43
44 44 wrapped_capabilities = lfproto.capabilities
45 45 logger = logging.getLogger('vcsserver.hg')
46 46
47 47 def _dynamic_capabilities(repo, proto):
48 48 """
49 49 Adds dynamic behavior, so that the capability is only added if the
50 50 extension is enabled in the current ui object.
51 51 """
52 52 if 'largefiles' in dict(extensions(repo.ui)):
53 53 logger.debug('Extension largefiles enabled')
54 54 calc_capabilities = wrapped_capabilities
55 55 else:
56 56 logger.debug('Extension largefiles disabled')
57 57 calc_capabilities = lfproto.capabilitiesorig
58 58 return calc_capabilities(repo, proto)
59 59
60 60 return _dynamic_capabilities
61
62
63 def patch_subrepo_type_mapping():
64 from collections import defaultdict
65 from hgcompat import subrepo
66 from exceptions import SubrepoMergeException
67
68 class NoOpSubrepo(subrepo.abstractsubrepo):
69
70 def __init__(self, ctx, path, *args, **kwargs):
71 """Initialize abstractsubrepo part
72
73 ``ctx`` is the context referring this subrepository in the
74 parent repository.
75
76 ``path`` is the path to this subrepository as seen from
77 innermost repository.
78 """
79 self.ui = ctx.repo().ui
80 self._ctx = ctx
81 self._path = path
82
83 def storeclean(self, path):
84 """
85 returns true if the repository has not changed since it was last
86 cloned from or pushed to a given repository.
87 """
88 return True
89
90 def dirty(self, ignoreupdate=False):
91 """returns true if the dirstate of the subrepo is dirty or does not
92 match current stored state. If ignoreupdate is true, only check
93 whether the subrepo has uncommitted changes in its dirstate.
94 """
95 return False
96
97 def basestate(self):
98 """current working directory base state, disregarding .hgsubstate
99 state and working directory modifications"""
100 substate = subrepo.state(self._ctx, self.ui)
101 file_system_path, rev, repotype = substate.get(self._path)
102 return rev
103
104 def remove(self):
105 """remove the subrepo
106
107 (should verify the dirstate is not dirty first)
108 """
109 pass
110
111 def get(self, state, overwrite=False):
112 """run whatever commands are needed to put the subrepo into
113 this state
114 """
115 pass
116
117 def merge(self, state):
118 """merge currently-saved state with the new state."""
119 raise SubrepoMergeException()
120
121 def push(self, opts):
122 """perform whatever action is analogous to 'hg push'
123
124 This may be a no-op on some systems.
125 """
126 pass
127
128 # Patch subrepo type mapping to always return our NoOpSubrepo class
129 # whenever a subrepo class is looked up.
130 subrepo.types = {
131 'hg': NoOpSubrepo,
132 'git': NoOpSubrepo,
133 'svn': NoOpSubrepo
134 }
Show file before | Show file after
@@ -1,358 +1,359 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2016 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import base64
19 19 import locale
20 20 import logging
21 21 import uuid
22 22 import wsgiref.util
23 23 from itertools import chain
24 24
25 25 import msgpack
26 26 from beaker.cache import CacheManager
27 27 from beaker.util import parse_cache_config_options
28 28 from pyramid.config import Configurator
29 29 from pyramid.wsgi import wsgiapp
30 30
31 31 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
32 32 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
33 33 from vcsserver.echo_stub.echo_app import EchoApp
34 34 from vcsserver.exceptions import HTTPRepoLocked
35 35 from vcsserver.server import VcsServer
36 36
37 37 try:
38 38 from vcsserver.git import GitFactory, GitRemote
39 39 except ImportError:
40 40 GitFactory = None
41 41 GitRemote = None
42 42 try:
43 43 from vcsserver.hg import MercurialFactory, HgRemote
44 44 except ImportError:
45 45 MercurialFactory = None
46 46 HgRemote = None
47 47 try:
48 48 from vcsserver.svn import SubversionFactory, SvnRemote
49 49 except ImportError:
50 50 SubversionFactory = None
51 51 SvnRemote = None
52 52
53 53 log = logging.getLogger(__name__)
54 54
55 55
56 56 class VCS(object):
57 57 def __init__(self, locale=None, cache_config=None):
58 58 self.locale = locale
59 59 self.cache_config = cache_config
60 60 self._configure_locale()
61 61 self._initialize_cache()
62 62
63 63 if GitFactory and GitRemote:
64 64 git_repo_cache = self.cache.get_cache_region(
65 65 'git', region='repo_object')
66 66 git_factory = GitFactory(git_repo_cache)
67 67 self._git_remote = GitRemote(git_factory)
68 68 else:
69 69 log.info("Git client import failed")
70 70
71 71 if MercurialFactory and HgRemote:
72 72 hg_repo_cache = self.cache.get_cache_region(
73 73 'hg', region='repo_object')
74 74 hg_factory = MercurialFactory(hg_repo_cache)
75 75 self._hg_remote = HgRemote(hg_factory)
76 76 else:
77 77 log.info("Mercurial client import failed")
78 78
79 79 if SubversionFactory and SvnRemote:
80 80 svn_repo_cache = self.cache.get_cache_region(
81 81 'svn', region='repo_object')
82 82 svn_factory = SubversionFactory(svn_repo_cache)
83 83 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
84 84 else:
85 85 log.info("Subversion client import failed")
86 86
87 87 self._vcsserver = VcsServer()
88 88
89 89 def _initialize_cache(self):
90 90 cache_config = parse_cache_config_options(self.cache_config)
91 91 log.info('Initializing beaker cache: %s' % cache_config)
92 92 self.cache = CacheManager(**cache_config)
93 93
94 94 def _configure_locale(self):
95 95 if self.locale:
96 96 log.info('Settings locale: `LC_ALL` to %s' % self.locale)
97 97 else:
98 98 log.info(
99 99 'Configuring locale subsystem based on environment variables')
100 100 try:
101 101 # If self.locale is the empty string, then the locale
102 102 # module will use the environment variables. See the
103 103 # documentation of the package `locale`.
104 104 locale.setlocale(locale.LC_ALL, self.locale)
105 105
106 106 language_code, encoding = locale.getlocale()
107 107 log.info(
108 108 'Locale set to language code "%s" with encoding "%s".',
109 109 language_code, encoding)
110 110 except locale.Error:
111 111 log.exception(
112 112 'Cannot set locale, not configuring the locale system')
113 113
114 114
115 115 class WsgiProxy(object):
116 116 def __init__(self, wsgi):
117 117 self.wsgi = wsgi
118 118
119 119 def __call__(self, environ, start_response):
120 120 input_data = environ['wsgi.input'].read()
121 121 input_data = msgpack.unpackb(input_data)
122 122
123 123 error = None
124 124 try:
125 125 data, status, headers = self.wsgi.handle(
126 126 input_data['environment'], input_data['input_data'],
127 127 *input_data['args'], **input_data['kwargs'])
128 128 except Exception as e:
129 129 data, status, headers = [], None, None
130 130 error = {
131 131 'message': str(e),
132 132 '_vcs_kind': getattr(e, '_vcs_kind', None)
133 133 }
134 134
135 135 start_response(200, {})
136 136 return self._iterator(error, status, headers, data)
137 137
138 138 def _iterator(self, error, status, headers, data):
139 139 initial_data = [
140 140 error,
141 141 status,
142 142 headers,
143 143 ]
144 144
145 145 for d in chain(initial_data, data):
146 146 yield msgpack.packb(d)
147 147
148 148
149 149 class HTTPApplication(object):
150 150 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
151 151
152 152 remote_wsgi = remote_wsgi
153 153 _use_echo_app = False
154 154
155 155 def __init__(self, settings=None):
156 156 self.config = Configurator(settings=settings)
157 157 locale = settings.get('', 'en_US.UTF-8')
158 158 vcs = VCS(locale=locale, cache_config=settings)
159 159 self._remotes = {
160 160 'hg': vcs._hg_remote,
161 161 'git': vcs._git_remote,
162 162 'svn': vcs._svn_remote,
163 163 'server': vcs._vcsserver,
164 164 }
165 165 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
166 166 self._use_echo_app = True
167 167 log.warning("Using EchoApp for VCS operations.")
168 168 self.remote_wsgi = remote_wsgi_stub
169 169 self._configure_settings(settings)
170 170 self._configure()
171 171
172 172 def _configure_settings(self, app_settings):
173 173 """
174 174 Configure the settings module.
175 175 """
176 176 git_path = app_settings.get('git_path', None)
177 177 if git_path:
178 178 settings.GIT_EXECUTABLE = git_path
179 179
180 180 def _configure(self):
181 181 self.config.add_renderer(
182 182 name='msgpack',
183 183 factory=self._msgpack_renderer_factory)
184 184
185 185 self.config.add_route('status', '/status')
186 186 self.config.add_route('hg_proxy', '/proxy/hg')
187 187 self.config.add_route('git_proxy', '/proxy/git')
188 188 self.config.add_route('vcs', '/{backend}')
189 189 self.config.add_route('stream_git', '/stream/git/*repo_name')
190 190 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
191 191
192 192 self.config.add_view(
193 193 self.status_view, route_name='status', renderer='json')
194 194 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
195 195 self.config.add_view(self.git_proxy(), route_name='git_proxy')
196 196 self.config.add_view(
197 197 self.vcs_view, route_name='vcs', renderer='msgpack')
198 198
199 199 self.config.add_view(self.hg_stream(), route_name='stream_hg')
200 200 self.config.add_view(self.git_stream(), route_name='stream_git')
201 201 self.config.add_view(
202 202 self.handle_vcs_exception, context=Exception,
203 203 custom_predicates=[self.is_vcs_exception])
204 204
205 205 def wsgi_app(self):
206 206 return self.config.make_wsgi_app()
207 207
208 208 def vcs_view(self, request):
209 209 remote = self._remotes[request.matchdict['backend']]
210 210 payload = msgpack.unpackb(request.body, use_list=True)
211 211 method = payload.get('method')
212 212 params = payload.get('params')
213 213 wire = params.get('wire')
214 214 args = params.get('args')
215 215 kwargs = params.get('kwargs')
216 216 if wire:
217 217 try:
218 218 wire['context'] = uuid.UUID(wire['context'])
219 219 except KeyError:
220 220 pass
221 221 args.insert(0, wire)
222 222
223 223 try:
224 224 resp = getattr(remote, method)(*args, **kwargs)
225 225 except Exception as e:
226 226 type_ = e.__class__.__name__
227 227 if type_ not in self.ALLOWED_EXCEPTIONS:
228 228 type_ = None
229 229
230 230 resp = {
231 231 'id': payload.get('id'),
232 232 'error': {
233 233 'message': e.message,
234 234 'type': type_
235 235 }
236 236 }
237 237 try:
238 238 resp['error']['_vcs_kind'] = e._vcs_kind
239 239 except AttributeError:
240 240 pass
241 241 else:
242 242 resp = {
243 243 'id': payload.get('id'),
244 244 'result': resp
245 245 }
246 246
247 247 return resp
248 248
249 249 def status_view(self, request):
250 250 return {'status': 'OK'}
251 251
252 252 def _msgpack_renderer_factory(self, info):
253 253 def _render(value, system):
254 254 value = msgpack.packb(value)
255 255 request = system.get('request')
256 256 if request is not None:
257 257 response = request.response
258 258 ct = response.content_type
259 259 if ct == response.default_content_type:
260 260 response.content_type = 'application/x-msgpack'
261 261 return value
262 262 return _render
263 263
264 264 def hg_proxy(self):
265 265 @wsgiapp
266 266 def _hg_proxy(environ, start_response):
267 267 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
268 268 return app(environ, start_response)
269 269 return _hg_proxy
270 270
271 271 def git_proxy(self):
272 272 @wsgiapp
273 273 def _git_proxy(environ, start_response):
274 274 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
275 275 return app(environ, start_response)
276 276 return _git_proxy
277 277
278 278 def hg_stream(self):
279 279 if self._use_echo_app:
280 280 @wsgiapp
281 281 def _hg_stream(environ, start_response):
282 282 app = EchoApp('fake_path', 'fake_name', None)
283 283 return app(environ, start_response)
284 284 return _hg_stream
285 285 else:
286 286 @wsgiapp
287 287 def _hg_stream(environ, start_response):
288 288 repo_path = environ['HTTP_X_RC_REPO_PATH']
289 289 repo_name = environ['HTTP_X_RC_REPO_NAME']
290 290 packed_config = base64.b64decode(
291 291 environ['HTTP_X_RC_REPO_CONFIG'])
292 292 config = msgpack.unpackb(packed_config)
293 293 app = scm_app.create_hg_wsgi_app(
294 294 repo_path, repo_name, config)
295 295
296 296 # Consitent path information for hgweb
297 297 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
298 298 environ['REPO_NAME'] = repo_name
299 299 return app(environ, ResponseFilter(start_response))
300 300 return _hg_stream
301 301
302 302 def git_stream(self):
303 303 if self._use_echo_app:
304 304 @wsgiapp
305 305 def _git_stream(environ, start_response):
306 306 app = EchoApp('fake_path', 'fake_name', None)
307 307 return app(environ, start_response)
308 308 return _git_stream
309 309 else:
310 310 @wsgiapp
311 311 def _git_stream(environ, start_response):
312 312 repo_path = environ['HTTP_X_RC_REPO_PATH']
313 313 repo_name = environ['HTTP_X_RC_REPO_NAME']
314 314 packed_config = base64.b64decode(
315 315 environ['HTTP_X_RC_REPO_CONFIG'])
316 316 config = msgpack.unpackb(packed_config)
317 317
318 318 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
319 319 app = scm_app.create_git_wsgi_app(
320 320 repo_path, repo_name, config)
321 321 return app(environ, start_response)
322 322 return _git_stream
323 323
324 324 def is_vcs_exception(self, context, request):
325 325 """
326 326 View predicate that returns true if the context object is a VCS
327 327 exception.
328 328 """
329 329 return hasattr(context, '_vcs_kind')
330 330
331 331 def handle_vcs_exception(self, exception, request):
332 332 if exception._vcs_kind == 'repo_locked':
333 333 # Get custom repo-locked status code if present.
334 334 status_code = request.headers.get('X-RC-Locked-Status-Code')
335 335 return HTTPRepoLocked(
336 336 title=exception.message, status_code=status_code)
337 337
338 338 # Re-raise exception if we can not handle it.
339 339 raise exception
340 340
341 341
342 342 class ResponseFilter(object):
343 343
344 344 def __init__(self, start_response):
345 345 self._start_response = start_response
346 346
347 347 def __call__(self, status, response_headers, exc_info=None):
348 348 headers = tuple(
349 349 (h, v) for h, v in response_headers
350 350 if not wsgiref.util.is_hop_by_hop(h))
351 351 return self._start_response(status, headers, exc_info)
352 352
353 353
354 354 def main(global_config, **settings):
355 355 if MercurialFactory:
356 356 hgpatches.patch_largefiles_capabilities()
357 hgpatches.patch_subrepo_type_mapping()
357 358 app = HTTPApplication(settings=settings)
358 359 return app.wsgi_app()
Show file before | Show file after
@@ -1,507 +1,508 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2016 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import atexit
19 19 import locale
20 20 import logging
21 21 import optparse
22 22 import os
23 23 import textwrap
24 24 import threading
25 25 import sys
26 26
27 27 import configobj
28 28 import Pyro4
29 29 from beaker.cache import CacheManager
30 30 from beaker.util import parse_cache_config_options
31 31
32 32 try:
33 33 from vcsserver.git import GitFactory, GitRemote
34 34 except ImportError:
35 35 GitFactory = None
36 36 GitRemote = None
37 37 try:
38 38 from vcsserver.hg import MercurialFactory, HgRemote
39 39 except ImportError:
40 40 MercurialFactory = None
41 41 HgRemote = None
42 42 try:
43 43 from vcsserver.svn import SubversionFactory, SvnRemote
44 44 except ImportError:
45 45 SubversionFactory = None
46 46 SvnRemote = None
47 47
48 48 from server import VcsServer
49 49 from vcsserver import hgpatches, remote_wsgi, settings
50 50 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
51 51
52 52 log = logging.getLogger(__name__)
53 53
54 54 HERE = os.path.dirname(os.path.abspath(__file__))
55 55 SERVER_RUNNING_FILE = None
56 56
57 57
58 58 # HOOKS - inspired by gunicorn #
59 59
60 60 def when_ready(server):
61 61 """
62 62 Called just after the server is started.
63 63 """
64 64
65 65 def _remove_server_running_file():
66 66 if os.path.isfile(SERVER_RUNNING_FILE):
67 67 os.remove(SERVER_RUNNING_FILE)
68 68
69 69 # top up to match to level location
70 70 if SERVER_RUNNING_FILE:
71 71 with open(SERVER_RUNNING_FILE, 'wb') as f:
72 72 f.write(str(os.getpid()))
73 73 # register cleanup of that file when server exits
74 74 atexit.register(_remove_server_running_file)
75 75
76 76
77 77 class LazyWriter(object):
78 78 """
79 79 File-like object that opens a file lazily when it is first written
80 80 to.
81 81 """
82 82
83 83 def __init__(self, filename, mode='w'):
84 84 self.filename = filename
85 85 self.fileobj = None
86 86 self.lock = threading.Lock()
87 87 self.mode = mode
88 88
89 89 def open(self):
90 90 if self.fileobj is None:
91 91 with self.lock:
92 92 self.fileobj = open(self.filename, self.mode)
93 93 return self.fileobj
94 94
95 95 def close(self):
96 96 fileobj = self.fileobj
97 97 if fileobj is not None:
98 98 fileobj.close()
99 99
100 100 def __del__(self):
101 101 self.close()
102 102
103 103 def write(self, text):
104 104 fileobj = self.open()
105 105 fileobj.write(text)
106 106 fileobj.flush()
107 107
108 108 def writelines(self, text):
109 109 fileobj = self.open()
110 110 fileobj.writelines(text)
111 111 fileobj.flush()
112 112
113 113 def flush(self):
114 114 self.open().flush()
115 115
116 116
117 117 class Application(object):
118 118 """
119 119 Represents the vcs server application.
120 120
121 121 This object is responsible to initialize the application and all needed
122 122 libraries. After that it hooks together the different objects and provides
123 123 them a way to access things like configuration.
124 124 """
125 125
126 126 def __init__(
127 127 self, host, port=None, locale='', threadpool_size=None,
128 128 timeout=None, cache_config=None, remote_wsgi_=None):
129 129
130 130 self.host = host
131 131 self.port = int(port) or settings.PYRO_PORT
132 132 self.threadpool_size = (
133 133 int(threadpool_size) if threadpool_size else None)
134 134 self.locale = locale
135 135 self.timeout = timeout
136 136 self.cache_config = cache_config
137 137 self.remote_wsgi = remote_wsgi_ or remote_wsgi
138 138
139 139 def init(self):
140 140 """
141 141 Configure and hook together all relevant objects.
142 142 """
143 143 self._configure_locale()
144 144 self._configure_pyro()
145 145 self._initialize_cache()
146 146 self._create_daemon_and_remote_objects(host=self.host, port=self.port)
147 147
148 148 def run(self):
149 149 """
150 150 Start the main loop of the application.
151 151 """
152 152
153 153 if hasattr(os, 'getpid'):
154 154 log.info('Starting %s in PID %i.', __name__, os.getpid())
155 155 else:
156 156 log.info('Starting %s.', __name__)
157 157 if SERVER_RUNNING_FILE:
158 158 log.info('PID file written as %s', SERVER_RUNNING_FILE)
159 159 else:
160 160 log.info('No PID file written by default.')
161 161 when_ready(self)
162 162 try:
163 163 self._pyrodaemon.requestLoop(
164 164 loopCondition=lambda: not self._vcsserver._shutdown)
165 165 finally:
166 166 self._pyrodaemon.shutdown()
167 167
168 168 def _configure_locale(self):
169 169 if self.locale:
170 170 log.info('Settings locale: `LC_ALL` to %s' % self.locale)
171 171 else:
172 172 log.info(
173 173 'Configuring locale subsystem based on environment variables')
174 174
175 175 try:
176 176 # If self.locale is the empty string, then the locale
177 177 # module will use the environment variables. See the
178 178 # documentation of the package `locale`.
179 179 locale.setlocale(locale.LC_ALL, self.locale)
180 180
181 181 language_code, encoding = locale.getlocale()
182 182 log.info(
183 183 'Locale set to language code "%s" with encoding "%s".',
184 184 language_code, encoding)
185 185 except locale.Error:
186 186 log.exception(
187 187 'Cannot set locale, not configuring the locale system')
188 188
189 189 def _configure_pyro(self):
190 190 if self.threadpool_size is not None:
191 191 log.info("Threadpool size set to %s", self.threadpool_size)
192 192 Pyro4.config.THREADPOOL_SIZE = self.threadpool_size
193 193 if self.timeout not in (None, 0, 0.0, '0'):
194 194 log.info("Timeout for RPC calls set to %s seconds", self.timeout)
195 195 Pyro4.config.COMMTIMEOUT = float(self.timeout)
196 196 Pyro4.config.SERIALIZER = 'pickle'
197 197 Pyro4.config.SERIALIZERS_ACCEPTED.add('pickle')
198 198 Pyro4.config.SOCK_REUSE = True
199 199 # Uncomment the next line when you need to debug remote errors
200 200 # Pyro4.config.DETAILED_TRACEBACK = True
201 201
202 202 def _initialize_cache(self):
203 203 cache_config = parse_cache_config_options(self.cache_config)
204 204 log.info('Initializing beaker cache: %s' % cache_config)
205 205 self.cache = CacheManager(**cache_config)
206 206
207 207 def _create_daemon_and_remote_objects(self, host='localhost',
208 208 port=settings.PYRO_PORT):
209 209 daemon = Pyro4.Daemon(host=host, port=port)
210 210
211 211 self._vcsserver = VcsServer()
212 212 uri = daemon.register(
213 213 self._vcsserver, objectId=settings.PYRO_VCSSERVER)
214 214 log.info("Object registered = %s", uri)
215 215
216 216 if GitFactory and GitRemote:
217 217 git_repo_cache = self.cache.get_cache_region('git', region='repo_object')
218 218 git_factory = GitFactory(git_repo_cache)
219 219 self._git_remote = GitRemote(git_factory)
220 220 uri = daemon.register(self._git_remote, objectId=settings.PYRO_GIT)
221 221 log.info("Object registered = %s", uri)
222 222 else:
223 223 log.info("Git client import failed")
224 224
225 225 if MercurialFactory and HgRemote:
226 226 hg_repo_cache = self.cache.get_cache_region('hg', region='repo_object')
227 227 hg_factory = MercurialFactory(hg_repo_cache)
228 228 self._hg_remote = HgRemote(hg_factory)
229 229 uri = daemon.register(self._hg_remote, objectId=settings.PYRO_HG)
230 230 log.info("Object registered = %s", uri)
231 231 else:
232 232 log.info("Mercurial client import failed")
233 233
234 234 if SubversionFactory and SvnRemote:
235 235 svn_repo_cache = self.cache.get_cache_region('svn', region='repo_object')
236 236 svn_factory = SubversionFactory(svn_repo_cache)
237 237 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
238 238 uri = daemon.register(self._svn_remote, objectId=settings.PYRO_SVN)
239 239 log.info("Object registered = %s", uri)
240 240 else:
241 241 log.info("Subversion client import failed")
242 242
243 243 self._git_remote_wsgi = self.remote_wsgi.GitRemoteWsgi()
244 244 uri = daemon.register(self._git_remote_wsgi,
245 245 objectId=settings.PYRO_GIT_REMOTE_WSGI)
246 246 log.info("Object registered = %s", uri)
247 247
248 248 self._hg_remote_wsgi = self.remote_wsgi.HgRemoteWsgi()
249 249 uri = daemon.register(self._hg_remote_wsgi,
250 250 objectId=settings.PYRO_HG_REMOTE_WSGI)
251 251 log.info("Object registered = %s", uri)
252 252
253 253 self._pyrodaemon = daemon
254 254
255 255
256 256 class VcsServerCommand(object):
257 257
258 258 usage = '%prog'
259 259 description = """
260 260 Runs the VCS server
261 261 """
262 262 default_verbosity = 1
263 263
264 264 parser = optparse.OptionParser(
265 265 usage,
266 266 description=textwrap.dedent(description)
267 267 )
268 268 parser.add_option(
269 269 '--host',
270 270 type="str",
271 271 dest="host",
272 272 )
273 273 parser.add_option(
274 274 '--port',
275 275 type="int",
276 276 dest="port"
277 277 )
278 278 parser.add_option(
279 279 '--running-file',
280 280 dest='running_file',
281 281 metavar='RUNNING_FILE',
282 282 help="Create a running file after the server is initalized with "
283 283 "stored PID of process"
284 284 )
285 285 parser.add_option(
286 286 '--locale',
287 287 dest='locale',
288 288 help="Allows to set the locale, e.g. en_US.UTF-8",
289 289 default=""
290 290 )
291 291 parser.add_option(
292 292 '--log-file',
293 293 dest='log_file',
294 294 metavar='LOG_FILE',
295 295 help="Save output to the given log file (redirects stdout)"
296 296 )
297 297 parser.add_option(
298 298 '--log-level',
299 299 dest="log_level",
300 300 metavar="LOG_LEVEL",
301 301 help="use LOG_LEVEL to set log level "
302 302 "(debug,info,warning,error,critical)"
303 303 )
304 304 parser.add_option(
305 305 '--threadpool',
306 306 dest='threadpool_size',
307 307 type='int',
308 308 help="Set the size of the threadpool used to communicate with the "
309 309 "WSGI workers. This should be at least 6 times the number of "
310 310 "WSGI worker processes."
311 311 )
312 312 parser.add_option(
313 313 '--timeout',
314 314 dest='timeout',
315 315 type='float',
316 316 help="Set the timeout for RPC communication in seconds."
317 317 )
318 318 parser.add_option(
319 319 '--config',
320 320 dest='config_file',
321 321 type='string',
322 322 help="Configuration file for vcsserver."
323 323 )
324 324
325 325 def __init__(self, argv, quiet=False):
326 326 self.options, self.args = self.parser.parse_args(argv[1:])
327 327 if quiet:
328 328 self.options.verbose = 0
329 329
330 330 def _get_file_config(self):
331 331 ini_conf = {}
332 332 conf = configobj.ConfigObj(self.options.config_file)
333 333 if 'DEFAULT' in conf:
334 334 ini_conf = conf['DEFAULT']
335 335
336 336 return ini_conf
337 337
338 338 def _show_config(self, vcsserver_config):
339 339 order = [
340 340 'config_file',
341 341 'host',
342 342 'port',
343 343 'log_file',
344 344 'log_level',
345 345 'locale',
346 346 'threadpool_size',
347 347 'timeout',
348 348 'cache_config',
349 349 ]
350 350
351 351 def sorter(k):
352 352 return dict([(y, x) for x, y in enumerate(order)]).get(k)
353 353
354 354 _config = []
355 355 for k in sorted(vcsserver_config.keys(), key=sorter):
356 356 v = vcsserver_config[k]
357 357 # construct padded key for display eg %-20s % = key: val
358 358 k_formatted = ('%-'+str(len(max(order, key=len))+1)+'s') % (k+':')
359 359 _config.append(' * %s %s' % (k_formatted, v))
360 360 log.info('\n[vcsserver configuration]:\n'+'\n'.join(_config))
361 361
362 362 def _get_vcsserver_configuration(self):
363 363 _defaults = {
364 364 'config_file': None,
365 365 'git_path': 'git',
366 366 'host': 'localhost',
367 367 'port': settings.PYRO_PORT,
368 368 'log_file': None,
369 369 'log_level': 'debug',
370 370 'locale': None,
371 371 'threadpool_size': 16,
372 372 'timeout': None,
373 373
374 374 # Development support
375 375 'dev.use_echo_app': False,
376 376
377 377 # caches, baker style config
378 378 'beaker.cache.regions': 'repo_object',
379 379 'beaker.cache.repo_object.expire': '10',
380 380 'beaker.cache.repo_object.type': 'memory',
381 381 }
382 382 config = {}
383 383 config.update(_defaults)
384 384 # overwrite defaults with one loaded from file
385 385 config.update(self._get_file_config())
386 386
387 387 # overwrite with self.option which has the top priority
388 388 for k, v in self.options.__dict__.items():
389 389 if v or v == 0:
390 390 config[k] = v
391 391
392 392 # clear all "extra" keys if they are somehow passed,
393 393 # we only want defaults, so any extra stuff from self.options is cleared
394 394 # except beaker stuff which needs to be dynamic
395 395 for k in [k for k in config.copy().keys() if not k.startswith('beaker.cache.')]:
396 396 if k not in _defaults:
397 397 del config[k]
398 398
399 399 # group together the cache into one key.
400 400 # Needed further for beaker lib configuration
401 401 _k = {}
402 402 for k in [k for k in config.copy() if k.startswith('beaker.cache.')]:
403 403 _k[k] = config.pop(k)
404 404 config['cache_config'] = _k
405 405
406 406 return config
407 407
408 408 def out(self, msg): # pragma: no cover
409 409 if self.options.verbose > 0:
410 410 print(msg)
411 411
412 412 def run(self): # pragma: no cover
413 413 vcsserver_config = self._get_vcsserver_configuration()
414 414
415 415 # Ensure the log file is writeable
416 416 if vcsserver_config['log_file']:
417 417 stdout_log = self._configure_logfile()
418 418 else:
419 419 stdout_log = None
420 420
421 421 # set PID file with running lock
422 422 if self.options.running_file:
423 423 global SERVER_RUNNING_FILE
424 424 SERVER_RUNNING_FILE = self.options.running_file
425 425
426 426 # configure logging, and logging based on configuration file
427 427 self._configure_logging(level=vcsserver_config['log_level'],
428 428 stream=stdout_log)
429 429 if self.options.config_file:
430 430 if not os.path.isfile(self.options.config_file):
431 431 raise OSError('File %s does not exist' %
432 432 self.options.config_file)
433 433
434 434 self._configure_file_logging(self.options.config_file)
435 435
436 436 self._configure_settings(vcsserver_config)
437 437
438 438 # display current configuration of vcsserver
439 439 self._show_config(vcsserver_config)
440 440
441 441 if not vcsserver_config['dev.use_echo_app']:
442 442 remote_wsgi_mod = remote_wsgi
443 443 else:
444 444 log.warning("Using EchoApp for VCS endpoints.")
445 445 remote_wsgi_mod = remote_wsgi_stub
446 446
447 447 app = Application(
448 448 host=vcsserver_config['host'],
449 449 port=vcsserver_config['port'],
450 450 locale=vcsserver_config['locale'],
451 451 threadpool_size=vcsserver_config['threadpool_size'],
452 452 timeout=vcsserver_config['timeout'],
453 453 cache_config=vcsserver_config['cache_config'],
454 454 remote_wsgi_=remote_wsgi_mod)
455 455 app.init()
456 456 app.run()
457 457
458 458 def _configure_logging(self, level, stream=None):
459 459 _format = (
460 460 '%(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s')
461 461 levels = {
462 462 'debug': logging.DEBUG,
463 463 'info': logging.INFO,
464 464 'warning': logging.WARNING,
465 465 'error': logging.ERROR,
466 466 'critical': logging.CRITICAL,
467 467 }
468 468 try:
469 469 level = levels[level]
470 470 except KeyError:
471 471 raise AttributeError(
472 472 'Invalid log level please use one of %s' % (levels.keys(),))
473 473 logging.basicConfig(format=_format, stream=stream, level=level)
474 474 logging.getLogger('Pyro4').setLevel(level)
475 475
476 476 def _configure_file_logging(self, config):
477 477 import logging.config
478 478 try:
479 479 logging.config.fileConfig(config)
480 480 except Exception as e:
481 481 log.warning('Failed to configure logging based on given '
482 482 'config file. Error: %s' % e)
483 483
484 484 def _configure_logfile(self):
485 485 try:
486 486 writeable_log_file = open(self.options.log_file, 'a')
487 487 except IOError as ioe:
488 488 msg = 'Error: Unable to write to log file: %s' % ioe
489 489 raise ValueError(msg)
490 490 writeable_log_file.close()
491 491 stdout_log = LazyWriter(self.options.log_file, 'a')
492 492 sys.stdout = stdout_log
493 493 sys.stderr = stdout_log
494 494 return stdout_log
495 495
496 496 def _configure_settings(self, config):
497 497 """
498 498 Configure the settings module based on the given `config`.
499 499 """
500 500 settings.GIT_EXECUTABLE = config['git_path']
501 501
502 502
503 503 def main(argv=sys.argv, quiet=False):
504 504 if MercurialFactory:
505 505 hgpatches.patch_largefiles_capabilities()
506 hgpatches.patch_subrepo_type_mapping()
506 507 command = VcsServerCommand(argv, quiet=quiet)
507 508 return command.run()