##// END OF EJS Templates
release: Merge default into stable for release preparation
marcink -
r156:c4fd9054 merge stable
parent child Browse files
Show More
@@ -0,0 +1,18 b''
1
2 .PHONY: clean test test-clean test-only
3
4
5 clean:
6 make test-clean
7 find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' \) -exec rm '{}' ';'
8
9 test:
10 make test-clean
11 make test-only
12
13 test-clean:
14 rm -rf coverage.xml htmlcov junit.xml pylint.log result
15 find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';'
16
17 test-only:
18 PYTHONHASHSEED=random py.test -vv -r xw --cov=vcsserver --cov-report=term-missing --cov-report=html vcsserver
@@ -0,0 +1,3 b''
1 [pytest]
2 testpaths = ./vcsserver
3 addopts = -v
@@ -0,0 +1,15 b''
1 # test related requirements
2 pytest==3.0.5
3 py==1.4.31
4 pytest-cov==2.4.0
5 pytest-sugar==0.7.1
6 pytest-runner==2.9.0
7 pytest-catchlog==1.2.2
8 pytest-profiling==1.2.2
9 gprof2dot==2016.10.13
10 pytest-timeout==1.2.0
11
12 mock==1.0.1
13 WebTest==1.4.3
14 cov-core==1.15.0
15 coverage==3.7.1
@@ -0,0 +1,2 b''
1 [aliases]
2 test = pytest
@@ -0,0 +1,60 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 #
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
18
19
20 import time
21 import logging
22
23
24 from vcsserver.utils import safe_str
25
26
27 log = logging.getLogger(__name__)
28
29
30 def get_access_path(request):
31 environ = request.environ
32 return environ.get('PATH_INFO')
33
34
35 class RequestWrapperTween(object):
36 def __init__(self, handler, registry):
37 self.handler = handler
38 self.registry = registry
39
40 # one-time configuration code goes here
41
42 def __call__(self, request):
43 start = time.time()
44 try:
45 response = self.handler(request)
46 finally:
47 end = time.time()
48
49 log.info('IP: %s Request to %s time: %.3fs' % (
50 '127.0.0.1',
51 safe_str(get_access_path(request)), end - start)
52 )
53
54 return response
55
56
57 def includeme(config):
58 config.add_tween(
59 'vcsserver.tweens.RequestWrapperTween',
60 )
@@ -1,6 +1,6 b''
1 1 [bumpversion]
2 current_version = 4.5.2
2 current_version = 4.6.0
3 3 message = release: Bump version {current_version} to {new_version}
4 4
5 5 [bumpversion:file:vcsserver/VERSION]
6 6
@@ -1,36 +1,38 b''
1 1 syntax: glob
2 2 *.orig
3 3 *.pyc
4 4 *.swp
5 *.sqlite
5 6 *.tox
6 7 *.egg-info
7 8 *.egg
9 *.eggs
8 10 *.idea
9 11 .DS_Store*
10 12
11 13
12 14 syntax: regexp
13 15
14 16 #.filename
15 17 ^\.settings$
16 18 ^\.project$
17 19 ^\.pydevproject$
18 20 ^\.coverage$
19 21 ^\.cache.*$
20 22 ^\.rhodecode$
21 23
22 24 ^_dev
23 25 ^._dev
24 26 ^build/
25 27 ^coverage\.xml$
26 28 ^data$
27 29 ^dev.ini$
28 30 ^acceptance_tests/dev.*\.ini$
29 31 ^dist/
30 32 ^fabfile.py
31 33 ^htmlcov
32 34 ^junit\.xml$
33 35 ^node_modules/
34 36 ^pylint.log$
35 37 ^build$
36 38 ^result$
@@ -1,16 +1,14 b''
1 1 [DEFAULT]
2 2 done = false
3 3
4 4 [task:bump_version]
5 5 done = true
6 6
7 7 [task:fixes_on_stable]
8 done = true
9 8
10 9 [task:pip2nix_generated]
11 done = true
12 10
13 11 [release]
14 state = prepared
15 version = 4.5.2
12 state = in_progress
13 version = 4.6.0
16 14
@@ -1,15 +1,13 b''
1 1 # top level files
2 include test.ini
3 include MANIFEST.in
4 include README.rst
5 include CHANGES.rst
6 include LICENSE.txt
2 include *.rst
3 include *.txt
7 4
5 # package extras
8 6 include vcsserver/VERSION
9 7
10 8 # all config files
11 9 recursive-include configs *
12 10
13 11 # skip any tests files
14 recursive-exclude tests *
12 recursive-exclude vcsserver/tests *
15 13
@@ -1,138 +1,153 b''
1 1 # Nix environment for the community edition
2 2 #
3 3 # This shall be as lean as possible, just producing the rhodecode-vcsserver
4 4 # derivation. For advanced tweaks to pimp up the development environment we use
5 5 # "shell.nix" so that it does not have to clutter this file.
6 6
7 7 { pkgs ? (import <nixpkgs> {})
8 8 , pythonPackages ? "python27Packages"
9 9 , pythonExternalOverrides ? self: super: {}
10 10 , doCheck ? true
11 11 }:
12 12
13 13 let pkgs_ = pkgs; in
14 14
15 15 let
16 16 pkgs = pkgs_.overridePackages (self: super: {
17 # bump GIT version
18 git = pkgs.lib.overrideDerivation pkgs_.git (oldAttrs: {
19 name = "git-2.9.3";
20 src = pkgs.fetchurl {
21 url = "https://www.kernel.org/pub/software/scm/git/git-2.9.3.tar.xz";
22 sha256 = "0qzs681a64k3shh5p0rg41l1z16fbk5sj0xga45k34hp1hsp654z";
23 };
24
25 });
26
17 27 # Override subversion derivation to
18 28 # - activate python bindings
19 29 subversion = let
20 30 subversionWithPython = super.subversion.override {
21 31 httpSupport = true;
22 32 pythonBindings = true;
23 33 python = self.python27Packages.python;
24 34 };
25 in pkgs.lib.overrideDerivation subversionWithPython (oldAttrs: {
35
36 in
37
38 pkgs.lib.overrideDerivation subversionWithPython (oldAttrs: {
26 39 patches = (oldAttrs.patches or []) ++
27 40 pkgs.lib.optionals pkgs.stdenv.isDarwin [
28 41 # johbo: "import svn.client" fails on darwin currently.
29 42 ./pkgs/subversion-1.9.4-darwin.patch
30 43 ];
31 44 });
45
32 46 });
33 47
34 48 inherit (pkgs.lib) fix extends;
35
36 49 basePythonPackages = with builtins; if isAttrs pythonPackages
37 50 then pythonPackages
38 51 else getAttr pythonPackages pkgs;
39 52
40 53 elem = builtins.elem;
41 54 basename = path: with pkgs.lib; last (splitString "/" path);
42 55 startsWith = prefix: full: let
43 56 actualPrefix = builtins.substring 0 (builtins.stringLength prefix) full;
44 57 in actualPrefix == prefix;
45 58
46 59 src-filter = path: type: with pkgs.lib;
47 60 let
48 61 ext = last (splitString "." path);
49 62 in
50 !elem (basename path) [
51 ".git" ".hg" "__pycache__" ".eggs" "node_modules"
52 "build" "data" "tmp"] &&
63 !elem (basename path) [".hg" ".git" "__pycache__" ".eggs"
64 "node_modules" "build" "data" "tmp"] &&
53 65 !elem ext ["egg-info" "pyc"] &&
54 66 !startsWith "result" path;
55 67
56 68 rhodecode-vcsserver-src = builtins.filterSource src-filter ./.;
57 69
58 70 pythonGeneratedPackages = self: basePythonPackages.override (a: {
59 71 inherit self;
60 })
61 // (scopedImport {
72 }) // (scopedImport {
62 73 self = self;
63 74 super = basePythonPackages;
64 75 inherit pkgs;
65 76 inherit (pkgs) fetchurl fetchgit;
66 77 } ./pkgs/python-packages.nix);
67 78
68 79 pythonOverrides = import ./pkgs/python-packages-overrides.nix {
69 inherit
70 basePythonPackages
71 pkgs;
80 inherit basePythonPackages pkgs;
72 81 };
73 82
74 83 version = builtins.readFile ./vcsserver/VERSION;
75 84
76 85 pythonLocalOverrides = self: super: {
77 86 rhodecode-vcsserver = super.rhodecode-vcsserver.override (attrs: {
78 inherit
79 doCheck
80 version;
87 inherit doCheck version;
88
81 89 name = "rhodecode-vcsserver-${version}";
82 90 releaseName = "RhodeCodeVCSServer-${version}";
83 91 src = rhodecode-vcsserver-src;
84 92
85 93 propagatedBuildInputs = attrs.propagatedBuildInputs ++ ([
86 94 pkgs.git
87 95 pkgs.subversion
88 96 ]);
89 97
90 98 # TODO: johbo: Make a nicer way to expose the parts. Maybe
91 99 # pkgs/default.nix?
92 100 passthru = {
93 101 pythonPackages = self;
94 102 };
95 103
96 104 # Add VCSServer bin directory to path so that tests can find 'vcsserver'.
97 105 preCheck = ''
98 106 export PATH="$out/bin:$PATH"
99 107 '';
100 108
109 # put custom attrs here
110 checkPhase = ''
111 runHook preCheck
112 PYTHONHASHSEED=random py.test -p no:sugar -vv --cov-config=.coveragerc --cov=vcsserver --cov-report=term-missing vcsserver
113 runHook postCheck
114 '';
115
101 116 postInstall = ''
102 117 echo "Writing meta information for rccontrol to nix-support/rccontrol"
103 118 mkdir -p $out/nix-support/rccontrol
104 119 cp -v vcsserver/VERSION $out/nix-support/rccontrol/version
105 120 echo "DONE: Meta information for rccontrol written"
106 121
107 122 ln -s ${self.pyramid}/bin/* $out/bin/
108 123 ln -s ${self.gunicorn}/bin/gunicorn $out/bin/
109 124
110 125 # Symlink version control utilities
111 126 #
112 127 # We ensure that always the correct version is available as a symlink.
113 128 # So that users calling them via the profile path will always use the
114 129 # correct version.
115 130 ln -s ${pkgs.git}/bin/git $out/bin
116 131 ln -s ${self.mercurial}/bin/hg $out/bin
117 132 ln -s ${pkgs.subversion}/bin/svn* $out/bin
118 133
119 134 for file in $out/bin/*; do
120 135 wrapProgram $file \
121 136 --set PATH $PATH \
122 137 --set PYTHONPATH $PYTHONPATH \
123 138 --set PYTHONHASHSEED random
124 139 done
125 140 '';
126 141
127 142 });
128 143 };
129 144
130 145 # Apply all overrides and fix the final package set
131 146 myPythonPackages =
132 147 (fix
133 148 (extends pythonExternalOverrides
134 149 (extends pythonLocalOverrides
135 150 (extends pythonOverrides
136 151 pythonGeneratedPackages))));
137 152
138 153 in myPythonPackages.rhodecode-vcsserver
@@ -1,57 +1,47 b''
1 1 # Overrides for the generated python-packages.nix
2 2 #
3 3 # This function is intended to be used as an extension to the generated file
4 4 # python-packages.nix. The main objective is to add needed dependencies of C
5 5 # libraries and tweak the build instructions where needed.
6 6
7 7 { pkgs, basePythonPackages }:
8 8
9 9 let
10 10 sed = "sed -i";
11 11 in
12 12
13 13 self: super: {
14 14
15 15 subvertpy = super.subvertpy.override (attrs: {
16 16 # TODO: johbo: Remove the "or" once we drop 16.03 support
17 17 SVN_PREFIX = "${pkgs.subversion.dev or pkgs.subversion}";
18 18 propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
19 19 pkgs.aprutil
20 20 pkgs.subversion
21 21 ];
22 22 preBuild = pkgs.lib.optionalString pkgs.stdenv.isDarwin ''
23 23 ${sed} -e "s/'gcc'/'clang'/" setup.py
24 24 '';
25 25 });
26 26
27 27 mercurial = super.mercurial.override (attrs: {
28 28 propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
29 29 self.python.modules.curses
30 30 ] ++ pkgs.lib.optional pkgs.stdenv.isDarwin
31 31 pkgs.darwin.apple_sdk.frameworks.ApplicationServices;
32 32 });
33 33
34 34 pyramid = super.pyramid.override (attrs: {
35 35 postFixup = ''
36 36 wrapPythonPrograms
37 37 # TODO: johbo: "wrapPython" adds this magic line which
38 38 # confuses pserve.
39 39 ${sed} '/import sys; sys.argv/d' $out/bin/.pserve-wrapped
40 40 '';
41 41 });
42 42
43 Pyro4 = super.Pyro4.override (attrs: {
44 # TODO: Was not able to generate this version, needs further
45 # investigation.
46 name = "Pyro4-4.35";
47 src = pkgs.fetchurl {
48 url = "https://pypi.python.org/packages/source/P/Pyro4/Pyro4-4.35.src.tar.gz";
49 md5 = "cbe6cb855f086a0f092ca075005855f3";
50 };
51 });
52
53 43 # Avoid that setuptools is replaced, this leads to trouble
54 44 # with buildPythonPackage.
55 45 setuptools = basePythonPackages.setuptools;
56 46
57 47 }
@@ -1,694 +1,812 b''
1 1 # Generated by pip2nix 0.4.0
2 2 # See https://github.com/johbo/pip2nix
3 3
4 4 {
5 5 Beaker = super.buildPythonPackage {
6 6 name = "Beaker-1.7.0";
7 7 buildInputs = with self; [];
8 8 doCheck = false;
9 9 propagatedBuildInputs = with self; [];
10 10 src = fetchurl {
11 11 url = "https://pypi.python.org/packages/97/8e/409d2e7c009b8aa803dc9e6f239f1db7c3cdf578249087a404e7c27a505d/Beaker-1.7.0.tar.gz";
12 12 md5 = "386be3f7fe427358881eee4622b428b3";
13 13 };
14 14 meta = {
15 15 license = [ pkgs.lib.licenses.bsdOriginal ];
16 16 };
17 17 };
18 18 Jinja2 = super.buildPythonPackage {
19 19 name = "Jinja2-2.8";
20 20 buildInputs = with self; [];
21 21 doCheck = false;
22 22 propagatedBuildInputs = with self; [MarkupSafe];
23 23 src = fetchurl {
24 24 url = "https://pypi.python.org/packages/f2/2f/0b98b06a345a761bec91a079ccae392d282690c2d8272e708f4d10829e22/Jinja2-2.8.tar.gz";
25 25 md5 = "edb51693fe22c53cee5403775c71a99e";
26 26 };
27 27 meta = {
28 28 license = [ pkgs.lib.licenses.bsdOriginal ];
29 29 };
30 30 };
31 31 Mako = super.buildPythonPackage {
32 32 name = "Mako-1.0.6";
33 33 buildInputs = with self; [];
34 34 doCheck = false;
35 35 propagatedBuildInputs = with self; [MarkupSafe];
36 36 src = fetchurl {
37 37 url = "https://pypi.python.org/packages/56/4b/cb75836863a6382199aefb3d3809937e21fa4cb0db15a4f4ba0ecc2e7e8e/Mako-1.0.6.tar.gz";
38 38 md5 = "a28e22a339080316b2acc352b9ee631c";
39 39 };
40 40 meta = {
41 41 license = [ pkgs.lib.licenses.mit ];
42 42 };
43 43 };
44 44 MarkupSafe = super.buildPythonPackage {
45 45 name = "MarkupSafe-0.23";
46 46 buildInputs = with self; [];
47 47 doCheck = false;
48 48 propagatedBuildInputs = with self; [];
49 49 src = fetchurl {
50 50 url = "https://pypi.python.org/packages/c0/41/bae1254e0396c0cc8cf1751cb7d9afc90a602353695af5952530482c963f/MarkupSafe-0.23.tar.gz";
51 51 md5 = "f5ab3deee4c37cd6a922fb81e730da6e";
52 52 };
53 53 meta = {
54 54 license = [ pkgs.lib.licenses.bsdOriginal ];
55 55 };
56 56 };
57 57 PasteDeploy = super.buildPythonPackage {
58 58 name = "PasteDeploy-1.5.2";
59 59 buildInputs = with self; [];
60 60 doCheck = false;
61 61 propagatedBuildInputs = with self; [];
62 62 src = fetchurl {
63 63 url = "https://pypi.python.org/packages/0f/90/8e20cdae206c543ea10793cbf4136eb9a8b3f417e04e40a29d72d9922cbd/PasteDeploy-1.5.2.tar.gz";
64 64 md5 = "352b7205c78c8de4987578d19431af3b";
65 65 };
66 66 meta = {
67 67 license = [ pkgs.lib.licenses.mit ];
68 68 };
69 69 };
70 70 Pyro4 = super.buildPythonPackage {
71 71 name = "Pyro4-4.41";
72 72 buildInputs = with self; [];
73 73 doCheck = false;
74 74 propagatedBuildInputs = with self; [serpent];
75 75 src = fetchurl {
76 76 url = "https://pypi.python.org/packages/56/2b/89b566b4bf3e7f8ba790db2d1223852f8cb454c52cab7693dd41f608ca2a/Pyro4-4.41.tar.gz";
77 77 md5 = "ed69e9bfafa9c06c049a87cb0c4c2b6c";
78 78 };
79 79 meta = {
80 80 license = [ pkgs.lib.licenses.mit ];
81 81 };
82 82 };
83 83 WebOb = super.buildPythonPackage {
84 84 name = "WebOb-1.3.1";
85 85 buildInputs = with self; [];
86 86 doCheck = false;
87 87 propagatedBuildInputs = with self; [];
88 88 src = fetchurl {
89 89 url = "https://pypi.python.org/packages/16/78/adfc0380b8a0d75b2d543fa7085ba98a573b1ae486d9def88d172b81b9fa/WebOb-1.3.1.tar.gz";
90 90 md5 = "20918251c5726956ba8fef22d1556177";
91 91 };
92 92 meta = {
93 93 license = [ pkgs.lib.licenses.mit ];
94 94 };
95 95 };
96 96 WebTest = super.buildPythonPackage {
97 97 name = "WebTest-1.4.3";
98 98 buildInputs = with self; [];
99 99 doCheck = false;
100 100 propagatedBuildInputs = with self; [WebOb];
101 101 src = fetchurl {
102 102 url = "https://pypi.python.org/packages/51/3d/84fd0f628df10b30c7db87895f56d0158e5411206b721ca903cb51bfd948/WebTest-1.4.3.zip";
103 103 md5 = "631ce728bed92c681a4020a36adbc353";
104 104 };
105 105 meta = {
106 106 license = [ pkgs.lib.licenses.mit ];
107 107 };
108 108 };
109 109 backports.shutil-get-terminal-size = super.buildPythonPackage {
110 110 name = "backports.shutil-get-terminal-size-1.0.0";
111 111 buildInputs = with self; [];
112 112 doCheck = false;
113 113 propagatedBuildInputs = with self; [];
114 114 src = fetchurl {
115 115 url = "https://pypi.python.org/packages/ec/9c/368086faa9c016efce5da3e0e13ba392c9db79e3ab740b763fe28620b18b/backports.shutil_get_terminal_size-1.0.0.tar.gz";
116 116 md5 = "03267762480bd86b50580dc19dff3c66";
117 117 };
118 118 meta = {
119 119 license = [ pkgs.lib.licenses.mit ];
120 120 };
121 121 };
122 122 configobj = super.buildPythonPackage {
123 123 name = "configobj-5.0.6";
124 124 buildInputs = with self; [];
125 125 doCheck = false;
126 126 propagatedBuildInputs = with self; [six];
127 127 src = fetchurl {
128 128 url = "https://pypi.python.org/packages/64/61/079eb60459c44929e684fa7d9e2fdca403f67d64dd9dbac27296be2e0fab/configobj-5.0.6.tar.gz";
129 129 md5 = "e472a3a1c2a67bb0ec9b5d54c13a47d6";
130 130 };
131 131 meta = {
132 132 license = [ pkgs.lib.licenses.bsdOriginal ];
133 133 };
134 134 };
135 cov-core = super.buildPythonPackage {
136 name = "cov-core-1.15.0";
137 buildInputs = with self; [];
138 doCheck = false;
139 propagatedBuildInputs = with self; [coverage];
140 src = fetchurl {
141 url = "https://pypi.python.org/packages/4b/87/13e75a47b4ba1be06f29f6d807ca99638bedc6b57fa491cd3de891ca2923/cov-core-1.15.0.tar.gz";
142 md5 = "f519d4cb4c4e52856afb14af52919fe6";
143 };
144 meta = {
145 license = [ pkgs.lib.licenses.mit ];
146 };
147 };
148 coverage = super.buildPythonPackage {
149 name = "coverage-3.7.1";
150 buildInputs = with self; [];
151 doCheck = false;
152 propagatedBuildInputs = with self; [];
153 src = fetchurl {
154 url = "https://pypi.python.org/packages/09/4f/89b06c7fdc09687bca507dc411c342556ef9c5a3b26756137a4878ff19bf/coverage-3.7.1.tar.gz";
155 md5 = "c47b36ceb17eaff3ecfab3bcd347d0df";
156 };
157 meta = {
158 license = [ pkgs.lib.licenses.bsdOriginal ];
159 };
160 };
135 161 decorator = super.buildPythonPackage {
136 162 name = "decorator-4.0.10";
137 163 buildInputs = with self; [];
138 164 doCheck = false;
139 165 propagatedBuildInputs = with self; [];
140 166 src = fetchurl {
141 167 url = "https://pypi.python.org/packages/13/8a/4eed41e338e8dcc13ca41c94b142d4d20c0de684ee5065523fee406ce76f/decorator-4.0.10.tar.gz";
142 168 md5 = "434b57fdc3230c500716c5aff8896100";
143 169 };
144 170 meta = {
145 171 license = [ pkgs.lib.licenses.bsdOriginal { fullName = "new BSD License"; } ];
146 172 };
147 173 };
148 174 dulwich = super.buildPythonPackage {
149 175 name = "dulwich-0.13.0";
150 176 buildInputs = with self; [];
151 177 doCheck = false;
152 178 propagatedBuildInputs = with self; [];
153 179 src = fetchurl {
154 180 url = "https://pypi.python.org/packages/84/95/732d280eee829dacc954e8109f97b47abcadcca472c2ab013e1635eb4792/dulwich-0.13.0.tar.gz";
155 181 md5 = "6dede0626657c2bd08f48ca1221eea91";
156 182 };
157 183 meta = {
158 184 license = [ pkgs.lib.licenses.gpl2Plus ];
159 185 };
160 186 };
161 187 enum34 = super.buildPythonPackage {
162 188 name = "enum34-1.1.6";
163 189 buildInputs = with self; [];
164 190 doCheck = false;
165 191 propagatedBuildInputs = with self; [];
166 192 src = fetchurl {
167 193 url = "https://pypi.python.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz";
168 194 md5 = "5f13a0841a61f7fc295c514490d120d0";
169 195 };
170 196 meta = {
171 197 license = [ pkgs.lib.licenses.bsdOriginal ];
172 198 };
173 199 };
174 greenlet = super.buildPythonPackage {
175 name = "greenlet-0.4.7";
200 gevent = super.buildPythonPackage {
201 name = "gevent-1.1.2";
202 buildInputs = with self; [];
203 doCheck = false;
204 propagatedBuildInputs = with self; [greenlet];
205 src = fetchurl {
206 url = "https://pypi.python.org/packages/43/8f/cb3224a0e6ab663547f45c10d0651cfd52633fde4283bf68d627084df8cc/gevent-1.1.2.tar.gz";
207 md5 = "bb32a2f852a4997138014d5007215c6e";
208 };
209 meta = {
210 license = [ pkgs.lib.licenses.mit ];
211 };
212 };
213 gprof2dot = super.buildPythonPackage {
214 name = "gprof2dot-2016.10.13";
176 215 buildInputs = with self; [];
177 216 doCheck = false;
178 217 propagatedBuildInputs = with self; [];
179 218 src = fetchurl {
180 url = "https://pypi.python.org/packages/7a/9f/a1a0d9bdf3203ae1502c5a8434fe89d323599d78a106985bc327351a69d4/greenlet-0.4.7.zip";
181 md5 = "c2333a8ff30fa75c5d5ec0e67b461086";
219 url = "https://pypi.python.org/packages/a0/e0/73c71baed306f0402a00a94ffc7b2be94ad1296dfcb8b46912655b93154c/gprof2dot-2016.10.13.tar.gz";
220 md5 = "0125401f15fd2afe1df686a76c64a4fd";
221 };
222 meta = {
223 license = [ { fullName = "LGPL"; } ];
224 };
225 };
226 greenlet = super.buildPythonPackage {
227 name = "greenlet-0.4.10";
228 buildInputs = with self; [];
229 doCheck = false;
230 propagatedBuildInputs = with self; [];
231 src = fetchurl {
232 url = "https://pypi.python.org/packages/67/62/ca2a95648666eaa2ffeb6a9b3964f21d419ae27f82f2e66b53da5b943fc4/greenlet-0.4.10.zip";
233 md5 = "bed0c4b3b896702131f4d5c72f87c41d";
182 234 };
183 235 meta = {
184 236 license = [ pkgs.lib.licenses.mit ];
185 237 };
186 238 };
187 239 gunicorn = super.buildPythonPackage {
188 240 name = "gunicorn-19.6.0";
189 241 buildInputs = with self; [];
190 242 doCheck = false;
191 243 propagatedBuildInputs = with self; [];
192 244 src = fetchurl {
193 245 url = "https://pypi.python.org/packages/84/ce/7ea5396efad1cef682bbc4068e72a0276341d9d9d0f501da609fab9fcb80/gunicorn-19.6.0.tar.gz";
194 246 md5 = "338e5e8a83ea0f0625f768dba4597530";
195 247 };
196 248 meta = {
197 249 license = [ pkgs.lib.licenses.mit ];
198 250 };
199 251 };
200 252 hgsubversion = super.buildPythonPackage {
201 253 name = "hgsubversion-1.8.6";
202 254 buildInputs = with self; [];
203 255 doCheck = false;
204 256 propagatedBuildInputs = with self; [mercurial subvertpy];
205 257 src = fetchurl {
206 258 url = "https://pypi.python.org/packages/ce/97/032e5093ad250e9908cea04395cbddb6902d587f712a79b53b2d778bdfdd/hgsubversion-1.8.6.tar.gz";
207 259 md5 = "9310cb266031cf8d0779885782a84a5b";
208 260 };
209 261 meta = {
210 262 license = [ pkgs.lib.licenses.gpl1 ];
211 263 };
212 264 };
213 265 infrae.cache = super.buildPythonPackage {
214 266 name = "infrae.cache-1.0.1";
215 267 buildInputs = with self; [];
216 268 doCheck = false;
217 269 propagatedBuildInputs = with self; [Beaker repoze.lru];
218 270 src = fetchurl {
219 271 url = "https://pypi.python.org/packages/bb/f0/e7d5e984cf6592fd2807dc7bc44a93f9d18e04e6a61f87fdfb2622422d74/infrae.cache-1.0.1.tar.gz";
220 272 md5 = "b09076a766747e6ed2a755cc62088e32";
221 273 };
222 274 meta = {
223 275 license = [ pkgs.lib.licenses.zpt21 ];
224 276 };
225 277 };
226 278 ipdb = super.buildPythonPackage {
227 279 name = "ipdb-0.10.1";
228 280 buildInputs = with self; [];
229 281 doCheck = false;
230 282 propagatedBuildInputs = with self; [ipython setuptools];
231 283 src = fetchurl {
232 284 url = "https://pypi.python.org/packages/eb/0a/0a37dc19572580336ad3813792c0d18c8d7117c2d66fc63c501f13a7a8f8/ipdb-0.10.1.tar.gz";
233 285 md5 = "4aeab65f633ddc98ebdb5eebf08dc713";
234 286 };
235 287 meta = {
236 288 license = [ pkgs.lib.licenses.bsdOriginal ];
237 289 };
238 290 };
239 291 ipython = super.buildPythonPackage {
240 292 name = "ipython-5.1.0";
241 293 buildInputs = with self; [];
242 294 doCheck = false;
243 295 propagatedBuildInputs = with self; [setuptools decorator pickleshare simplegeneric traitlets prompt-toolkit pygments pexpect backports.shutil-get-terminal-size pathlib2 pexpect];
244 296 src = fetchurl {
245 297 url = "https://pypi.python.org/packages/89/63/a9292f7cd9d0090a0f995e1167f3f17d5889dcbc9a175261719c513b9848/ipython-5.1.0.tar.gz";
246 298 md5 = "47c8122420f65b58784cb4b9b4af35e3";
247 299 };
248 300 meta = {
249 301 license = [ pkgs.lib.licenses.bsdOriginal ];
250 302 };
251 303 };
252 304 ipython-genutils = super.buildPythonPackage {
253 305 name = "ipython-genutils-0.1.0";
254 306 buildInputs = with self; [];
255 307 doCheck = false;
256 308 propagatedBuildInputs = with self; [];
257 309 src = fetchurl {
258 310 url = "https://pypi.python.org/packages/71/b7/a64c71578521606edbbce15151358598f3dfb72a3431763edc2baf19e71f/ipython_genutils-0.1.0.tar.gz";
259 311 md5 = "9a8afbe0978adbcbfcb3b35b2d015a56";
260 312 };
261 313 meta = {
262 314 license = [ pkgs.lib.licenses.bsdOriginal ];
263 315 };
264 316 };
265 317 mercurial = super.buildPythonPackage {
266 name = "mercurial-3.8.4";
318 name = "mercurial-4.0.2";
267 319 buildInputs = with self; [];
268 320 doCheck = false;
269 321 propagatedBuildInputs = with self; [];
270 322 src = fetchurl {
271 url = "https://pypi.python.org/packages/bc/16/b66eef0b70ee2b4ebb8e76622fe21bbed834606dd8c1bd30d6936ebf6f45/mercurial-3.8.4.tar.gz";
272 md5 = "cec2c3db688cb87142809089c6ae13e9";
323 url = "https://pypi.python.org/packages/85/1b/0296aacd697228974a473d2508f013532f987ed6b1bacfe5abd6d5be6332/mercurial-4.0.2.tar.gz";
324 md5 = "fa72a08e2723e4fa2a21c4e66437f3fa";
273 325 };
274 326 meta = {
275 327 license = [ pkgs.lib.licenses.gpl1 pkgs.lib.licenses.gpl2Plus ];
276 328 };
277 329 };
278 330 mock = super.buildPythonPackage {
279 331 name = "mock-1.0.1";
280 332 buildInputs = with self; [];
281 333 doCheck = false;
282 334 propagatedBuildInputs = with self; [];
283 335 src = fetchurl {
284 336 url = "https://pypi.python.org/packages/15/45/30273ee91feb60dabb8fbb2da7868520525f02cf910279b3047182feed80/mock-1.0.1.zip";
285 337 md5 = "869f08d003c289a97c1a6610faf5e913";
286 338 };
287 339 meta = {
288 340 license = [ pkgs.lib.licenses.bsdOriginal ];
289 341 };
290 342 };
291 343 msgpack-python = super.buildPythonPackage {
292 name = "msgpack-python-0.4.6";
344 name = "msgpack-python-0.4.8";
293 345 buildInputs = with self; [];
294 346 doCheck = false;
295 347 propagatedBuildInputs = with self; [];
296 348 src = fetchurl {
297 url = "https://pypi.python.org/packages/15/ce/ff2840885789ef8035f66cd506ea05bdb228340307d5e71a7b1e3f82224c/msgpack-python-0.4.6.tar.gz";
298 md5 = "8b317669314cf1bc881716cccdaccb30";
349 url = "https://pypi.python.org/packages/21/27/8a1d82041c7a2a51fcc73675875a5f9ea06c2663e02fcfeb708be1d081a0/msgpack-python-0.4.8.tar.gz";
350 md5 = "dcd854fb41ee7584ebbf35e049e6be98";
299 351 };
300 352 meta = {
301 353 license = [ pkgs.lib.licenses.asl20 ];
302 354 };
303 355 };
304 356 pathlib2 = super.buildPythonPackage {
305 357 name = "pathlib2-2.1.0";
306 358 buildInputs = with self; [];
307 359 doCheck = false;
308 360 propagatedBuildInputs = with self; [six];
309 361 src = fetchurl {
310 362 url = "https://pypi.python.org/packages/c9/27/8448b10d8440c08efeff0794adf7d0ed27adb98372c70c7b38f3947d4749/pathlib2-2.1.0.tar.gz";
311 363 md5 = "38e4f58b4d69dfcb9edb49a54a8b28d2";
312 364 };
313 365 meta = {
314 366 license = [ pkgs.lib.licenses.mit ];
315 367 };
316 368 };
317 369 pexpect = super.buildPythonPackage {
318 370 name = "pexpect-4.2.1";
319 371 buildInputs = with self; [];
320 372 doCheck = false;
321 373 propagatedBuildInputs = with self; [ptyprocess];
322 374 src = fetchurl {
323 375 url = "https://pypi.python.org/packages/e8/13/d0b0599099d6cd23663043a2a0bb7c61e58c6ba359b2656e6fb000ef5b98/pexpect-4.2.1.tar.gz";
324 376 md5 = "3694410001a99dff83f0b500a1ca1c95";
325 377 };
326 378 meta = {
327 379 license = [ pkgs.lib.licenses.isc { fullName = "ISC License (ISCL)"; } ];
328 380 };
329 381 };
330 382 pickleshare = super.buildPythonPackage {
331 383 name = "pickleshare-0.7.4";
332 384 buildInputs = with self; [];
333 385 doCheck = false;
334 386 propagatedBuildInputs = with self; [pathlib2];
335 387 src = fetchurl {
336 388 url = "https://pypi.python.org/packages/69/fe/dd137d84daa0fd13a709e448138e310d9ea93070620c9db5454e234af525/pickleshare-0.7.4.tar.gz";
337 389 md5 = "6a9e5dd8dfc023031f6b7b3f824cab12";
338 390 };
339 391 meta = {
340 392 license = [ pkgs.lib.licenses.mit ];
341 393 };
342 394 };
343 395 prompt-toolkit = super.buildPythonPackage {
344 396 name = "prompt-toolkit-1.0.9";
345 397 buildInputs = with self; [];
346 398 doCheck = false;
347 399 propagatedBuildInputs = with self; [six wcwidth];
348 400 src = fetchurl {
349 401 url = "https://pypi.python.org/packages/83/14/5ac258da6c530eca02852ee25c7a9ff3ca78287bb4c198d0d0055845d856/prompt_toolkit-1.0.9.tar.gz";
350 402 md5 = "a39f91a54308fb7446b1a421c11f227c";
351 403 };
352 404 meta = {
353 405 license = [ pkgs.lib.licenses.bsdOriginal ];
354 406 };
355 407 };
356 408 ptyprocess = super.buildPythonPackage {
357 409 name = "ptyprocess-0.5.1";
358 410 buildInputs = with self; [];
359 411 doCheck = false;
360 412 propagatedBuildInputs = with self; [];
361 413 src = fetchurl {
362 414 url = "https://pypi.python.org/packages/db/d7/b465161910f3d1cef593c5e002bff67e0384898f597f1a7fdc8db4c02bf6/ptyprocess-0.5.1.tar.gz";
363 415 md5 = "94e537122914cc9ec9c1eadcd36e73a1";
364 416 };
365 417 meta = {
366 418 license = [ ];
367 419 };
368 420 };
369 421 py = super.buildPythonPackage {
370 name = "py-1.4.29";
422 name = "py-1.4.31";
371 423 buildInputs = with self; [];
372 424 doCheck = false;
373 425 propagatedBuildInputs = with self; [];
374 426 src = fetchurl {
375 url = "https://pypi.python.org/packages/2a/bc/a1a4a332ac10069b8e5e25136a35e08a03f01fd6ab03d819889d79a1fd65/py-1.4.29.tar.gz";
376 md5 = "c28e0accba523a29b35a48bb703fb96c";
427 url = "https://pypi.python.org/packages/f4/9a/8dfda23f36600dd701c6722316ba8a3ab4b990261f83e7d3ffc6dfedf7ef/py-1.4.31.tar.gz";
428 md5 = "5d2c63c56dc3f2115ec35c066ecd582b";
377 429 };
378 430 meta = {
379 431 license = [ pkgs.lib.licenses.mit ];
380 432 };
381 433 };
382 434 pygments = super.buildPythonPackage {
383 435 name = "pygments-2.1.3";
384 436 buildInputs = with self; [];
385 437 doCheck = false;
386 438 propagatedBuildInputs = with self; [];
387 439 src = fetchurl {
388 440 url = "https://pypi.python.org/packages/b8/67/ab177979be1c81bc99c8d0592ef22d547e70bb4c6815c383286ed5dec504/Pygments-2.1.3.tar.gz";
389 441 md5 = "ed3fba2467c8afcda4d317e4ef2c6150";
390 442 };
391 443 meta = {
392 444 license = [ pkgs.lib.licenses.bsdOriginal ];
393 445 };
394 446 };
395 447 pyramid = super.buildPythonPackage {
396 448 name = "pyramid-1.6.1";
397 449 buildInputs = with self; [];
398 450 doCheck = false;
399 451 propagatedBuildInputs = with self; [setuptools WebOb repoze.lru zope.interface zope.deprecation venusian translationstring PasteDeploy];
400 452 src = fetchurl {
401 453 url = "https://pypi.python.org/packages/30/b3/fcc4a2a4800cbf21989e00454b5828cf1f7fe35c63e0810b350e56d4c475/pyramid-1.6.1.tar.gz";
402 454 md5 = "b18688ff3cc33efdbb098a35b45dd122";
403 455 };
404 456 meta = {
405 457 license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
406 458 };
407 459 };
408 460 pyramid-jinja2 = super.buildPythonPackage {
409 461 name = "pyramid-jinja2-2.5";
410 462 buildInputs = with self; [];
411 463 doCheck = false;
412 464 propagatedBuildInputs = with self; [pyramid zope.deprecation Jinja2 MarkupSafe];
413 465 src = fetchurl {
414 466 url = "https://pypi.python.org/packages/a1/80/595e26ffab7deba7208676b6936b7e5a721875710f982e59899013cae1ed/pyramid_jinja2-2.5.tar.gz";
415 467 md5 = "07cb6547204ac5e6f0b22a954ccee928";
416 468 };
417 469 meta = {
418 470 license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
419 471 };
420 472 };
421 473 pyramid-mako = super.buildPythonPackage {
422 474 name = "pyramid-mako-1.0.2";
423 475 buildInputs = with self; [];
424 476 doCheck = false;
425 477 propagatedBuildInputs = with self; [pyramid Mako];
426 478 src = fetchurl {
427 479 url = "https://pypi.python.org/packages/f1/92/7e69bcf09676d286a71cb3bbb887b16595b96f9ba7adbdc239ffdd4b1eb9/pyramid_mako-1.0.2.tar.gz";
428 480 md5 = "ee25343a97eb76bd90abdc2a774eb48a";
429 481 };
430 482 meta = {
431 483 license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
432 484 };
433 485 };
434 486 pytest = super.buildPythonPackage {
435 name = "pytest-2.8.5";
487 name = "pytest-3.0.5";
436 488 buildInputs = with self; [];
437 489 doCheck = false;
438 490 propagatedBuildInputs = with self; [py];
439 491 src = fetchurl {
440 url = "https://pypi.python.org/packages/b1/3d/d7ea9b0c51e0cacded856e49859f0a13452747491e842c236bbab3714afe/pytest-2.8.5.zip";
441 md5 = "8493b06f700862f1294298d6c1b715a9";
492 url = "https://pypi.python.org/packages/a8/87/b7ca49efe52d2b4169f2bfc49aa5e384173c4619ea8e635f123a0dac5b75/pytest-3.0.5.tar.gz";
493 md5 = "cefd527b59332688bf5db4a10aa8a7cb";
494 };
495 meta = {
496 license = [ pkgs.lib.licenses.mit ];
497 };
498 };
499 pytest-catchlog = super.buildPythonPackage {
500 name = "pytest-catchlog-1.2.2";
501 buildInputs = with self; [];
502 doCheck = false;
503 propagatedBuildInputs = with self; [py pytest];
504 src = fetchurl {
505 url = "https://pypi.python.org/packages/f2/2b/2faccdb1a978fab9dd0bf31cca9f6847fbe9184a0bdcc3011ac41dd44191/pytest-catchlog-1.2.2.zip";
506 md5 = "09d890c54c7456c818102b7ff8c182c8";
442 507 };
443 508 meta = {
444 509 license = [ pkgs.lib.licenses.mit ];
445 510 };
446 511 };
512 pytest-cov = super.buildPythonPackage {
513 name = "pytest-cov-2.4.0";
514 buildInputs = with self; [];
515 doCheck = false;
516 propagatedBuildInputs = with self; [pytest coverage];
517 src = fetchurl {
518 url = "https://pypi.python.org/packages/00/c0/2bfd1fcdb9d407b8ac8185b1cb5ff458105c6b207a9a7f0e13032de9828f/pytest-cov-2.4.0.tar.gz";
519 md5 = "2fda09677d232acc99ec1b3c5831e33f";
520 };
521 meta = {
522 license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.mit ];
523 };
524 };
525 pytest-profiling = super.buildPythonPackage {
526 name = "pytest-profiling-1.2.2";
527 buildInputs = with self; [];
528 doCheck = false;
529 propagatedBuildInputs = with self; [six pytest gprof2dot];
530 src = fetchurl {
531 url = "https://pypi.python.org/packages/73/e8/804681323bac0bc45c520ec34185ba8469008942266d0074699b204835c1/pytest-profiling-1.2.2.tar.gz";
532 md5 = "0a16d7dda2d23b91e9730fa4558cf728";
533 };
534 meta = {
535 license = [ pkgs.lib.licenses.mit ];
536 };
537 };
538 pytest-runner = super.buildPythonPackage {
539 name = "pytest-runner-2.9";
540 buildInputs = with self; [];
541 doCheck = false;
542 propagatedBuildInputs = with self; [];
543 src = fetchurl {
544 url = "https://pypi.python.org/packages/11/d4/c335ddf94463e451109e3494e909765c3e5205787b772e3b25ee8601b86a/pytest-runner-2.9.tar.gz";
545 md5 = "2212a2e34404b0960b2fdc2c469247b2";
546 };
547 meta = {
548 license = [ pkgs.lib.licenses.mit ];
549 };
550 };
551 pytest-sugar = super.buildPythonPackage {
552 name = "pytest-sugar-0.7.1";
553 buildInputs = with self; [];
554 doCheck = false;
555 propagatedBuildInputs = with self; [pytest termcolor];
556 src = fetchurl {
557 url = "https://pypi.python.org/packages/03/97/05d988b4fa870e7373e8ee4582408543b9ca2bd35c3c67b569369c6f9c49/pytest-sugar-0.7.1.tar.gz";
558 md5 = "7400f7c11f3d572b2c2a3b60352d35fe";
559 };
560 meta = {
561 license = [ pkgs.lib.licenses.bsdOriginal ];
562 };
563 };
564 pytest-timeout = super.buildPythonPackage {
565 name = "pytest-timeout-1.2.0";
566 buildInputs = with self; [];
567 doCheck = false;
568 propagatedBuildInputs = with self; [pytest];
569 src = fetchurl {
570 url = "https://pypi.python.org/packages/cc/b7/b2a61365ea6b6d2e8881360ae7ed8dad0327ad2df89f2f0be4a02304deb2/pytest-timeout-1.2.0.tar.gz";
571 md5 = "83607d91aa163562c7ee835da57d061d";
572 };
573 meta = {
574 license = [ pkgs.lib.licenses.mit { fullName = "DFSG approved"; } ];
575 };
576 };
447 577 repoze.lru = super.buildPythonPackage {
448 578 name = "repoze.lru-0.6";
449 579 buildInputs = with self; [];
450 580 doCheck = false;
451 581 propagatedBuildInputs = with self; [];
452 582 src = fetchurl {
453 583 url = "https://pypi.python.org/packages/6e/1e/aa15cc90217e086dc8769872c8778b409812ff036bf021b15795638939e4/repoze.lru-0.6.tar.gz";
454 584 md5 = "2c3b64b17a8e18b405f55d46173e14dd";
455 585 };
456 586 meta = {
457 587 license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
458 588 };
459 589 };
460 590 rhodecode-vcsserver = super.buildPythonPackage {
461 name = "rhodecode-vcsserver-4.5.2";
462 buildInputs = with self; [mock pytest pytest-sugar WebTest];
591 name = "rhodecode-vcsserver-4.6.0";
592 buildInputs = with self; [pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage configobj];
463 593 doCheck = true;
464 propagatedBuildInputs = with self; [configobj dulwich hgsubversion infrae.cache mercurial msgpack-python pyramid Pyro4 simplejson subprocess32 waitress WebOb];
594 propagatedBuildInputs = with self; [Beaker configobj dulwich hgsubversion infrae.cache mercurial msgpack-python pyramid pyramid-jinja2 pyramid-mako repoze.lru simplejson subprocess32 subvertpy six translationstring WebOb wheel zope.deprecation zope.interface ipdb gevent greenlet gunicorn waitress Pyro4 serpent pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage];
465 595 src = ./.;
466 596 meta = {
467 license = [ pkgs.lib.licenses.gpl3 { fullName = "GNU General Public License v3 or later (GPLv3+)"; } ];
597 license = [ { fullName = "GPL V3"; } { fullName = "GNU General Public License v3 or later (GPLv3+)"; } ];
468 598 };
469 599 };
470 600 serpent = super.buildPythonPackage {
471 name = "serpent-1.12";
601 name = "serpent-1.15";
472 602 buildInputs = with self; [];
473 603 doCheck = false;
474 604 propagatedBuildInputs = with self; [];
475 605 src = fetchurl {
476 url = "https://pypi.python.org/packages/3b/19/1e0e83b47c09edaef8398655088036e7e67386b5c48770218ebb339fbbd5/serpent-1.12.tar.gz";
477 md5 = "05869ac7b062828b34f8f927f0457b65";
606 url = "https://pypi.python.org/packages/7b/38/b2b27673a882ff2ea5871bb3e3e6b496ebbaafd1612e51990ffb158b9254/serpent-1.15.tar.gz";
607 md5 = "e27b1aad5c218e16442f52abb7c7053a";
478 608 };
479 609 meta = {
480 610 license = [ pkgs.lib.licenses.mit ];
481 611 };
482 612 };
483 613 setuptools = super.buildPythonPackage {
484 name = "setuptools-20.8.1";
614 name = "setuptools-30.1.0";
485 615 buildInputs = with self; [];
486 616 doCheck = false;
487 617 propagatedBuildInputs = with self; [];
488 618 src = fetchurl {
489 url = "https://pypi.python.org/packages/c4/19/c1bdc88b53da654df43770f941079dbab4e4788c2dcb5658fb86259894c7/setuptools-20.8.1.zip";
490 md5 = "fe58a5cac0df20bb83942b252a4b0543";
619 url = "https://pypi.python.org/packages/1e/43/002c8616db9a3e7be23c2556e39b90a32bb40ba0dc652de1999d5334d372/setuptools-30.1.0.tar.gz";
620 md5 = "cac497f42e5096ac8df29e38d3f81c3e";
491 621 };
492 622 meta = {
493 623 license = [ pkgs.lib.licenses.mit ];
494 624 };
495 625 };
496 626 simplegeneric = super.buildPythonPackage {
497 627 name = "simplegeneric-0.8.1";
498 628 buildInputs = with self; [];
499 629 doCheck = false;
500 630 propagatedBuildInputs = with self; [];
501 631 src = fetchurl {
502 632 url = "https://pypi.python.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip";
503 633 md5 = "f9c1fab00fd981be588fc32759f474e3";
504 634 };
505 635 meta = {
506 636 license = [ pkgs.lib.licenses.zpt21 ];
507 637 };
508 638 };
509 639 simplejson = super.buildPythonPackage {
510 640 name = "simplejson-3.7.2";
511 641 buildInputs = with self; [];
512 642 doCheck = false;
513 643 propagatedBuildInputs = with self; [];
514 644 src = fetchurl {
515 645 url = "https://pypi.python.org/packages/6d/89/7f13f099344eea9d6722779a1f165087cb559598107844b1ac5dbd831fb1/simplejson-3.7.2.tar.gz";
516 646 md5 = "a5fc7d05d4cb38492285553def5d4b46";
517 647 };
518 648 meta = {
519 649 license = [ { fullName = "Academic Free License (AFL)"; } pkgs.lib.licenses.mit ];
520 650 };
521 651 };
522 652 six = super.buildPythonPackage {
523 653 name = "six-1.9.0";
524 654 buildInputs = with self; [];
525 655 doCheck = false;
526 656 propagatedBuildInputs = with self; [];
527 657 src = fetchurl {
528 658 url = "https://pypi.python.org/packages/16/64/1dc5e5976b17466fd7d712e59cbe9fb1e18bec153109e5ba3ed6c9102f1a/six-1.9.0.tar.gz";
529 659 md5 = "476881ef4012262dfc8adc645ee786c4";
530 660 };
531 661 meta = {
532 662 license = [ pkgs.lib.licenses.mit ];
533 663 };
534 664 };
535 665 subprocess32 = super.buildPythonPackage {
536 666 name = "subprocess32-3.2.6";
537 667 buildInputs = with self; [];
538 668 doCheck = false;
539 669 propagatedBuildInputs = with self; [];
540 670 src = fetchurl {
541 671 url = "https://pypi.python.org/packages/28/8d/33ccbff51053f59ae6c357310cac0e79246bbed1d345ecc6188b176d72c3/subprocess32-3.2.6.tar.gz";
542 672 md5 = "754c5ab9f533e764f931136974b618f1";
543 673 };
544 674 meta = {
545 675 license = [ pkgs.lib.licenses.psfl ];
546 676 };
547 677 };
548 678 subvertpy = super.buildPythonPackage {
549 679 name = "subvertpy-0.9.3";
550 680 buildInputs = with self; [];
551 681 doCheck = false;
552 682 propagatedBuildInputs = with self; [];
553 683 src = fetchurl {
554 url = "https://github.com/jelmer/subvertpy/archive/subvertpy-0.9.3.tar.gz";
555 md5 = "7b745a47128050ea5a73efcd913ec1cf";
684 url = "https://code.rhodecode.com/upstream/subvertpy/archive/subvertpy-0.9.3.tar.gz?md5=4e49da2fe07608239cc9a80a7bb8f33c";
685 md5 = "4e49da2fe07608239cc9a80a7bb8f33c";
556 686 };
557 687 meta = {
558 688 license = [ pkgs.lib.licenses.lgpl21Plus ];
559 689 };
560 690 };
691 termcolor = super.buildPythonPackage {
692 name = "termcolor-1.1.0";
693 buildInputs = with self; [];
694 doCheck = false;
695 propagatedBuildInputs = with self; [];
696 src = fetchurl {
697 url = "https://pypi.python.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz";
698 md5 = "043e89644f8909d462fbbfa511c768df";
699 };
700 meta = {
701 license = [ pkgs.lib.licenses.mit ];
702 };
703 };
561 704 traitlets = super.buildPythonPackage {
562 705 name = "traitlets-4.3.1";
563 706 buildInputs = with self; [];
564 707 doCheck = false;
565 708 propagatedBuildInputs = with self; [ipython-genutils six decorator enum34];
566 709 src = fetchurl {
567 710 url = "https://pypi.python.org/packages/b1/d6/5b5aa6d5c474691909b91493da1e8972e309c9f01ecfe4aeafd272eb3234/traitlets-4.3.1.tar.gz";
568 711 md5 = "dd0b1b6e5d31ce446d55a4b5e5083c98";
569 712 };
570 713 meta = {
571 714 license = [ pkgs.lib.licenses.bsdOriginal ];
572 715 };
573 716 };
574 717 translationstring = super.buildPythonPackage {
575 718 name = "translationstring-1.3";
576 719 buildInputs = with self; [];
577 720 doCheck = false;
578 721 propagatedBuildInputs = with self; [];
579 722 src = fetchurl {
580 723 url = "https://pypi.python.org/packages/5e/eb/bee578cc150b44c653b63f5ebe258b5d0d812ddac12497e5f80fcad5d0b4/translationstring-1.3.tar.gz";
581 724 md5 = "a4b62e0f3c189c783a1685b3027f7c90";
582 725 };
583 726 meta = {
584 727 license = [ { fullName = "BSD-like (http://repoze.org/license.html)"; } ];
585 728 };
586 729 };
587 730 venusian = super.buildPythonPackage {
588 731 name = "venusian-1.0";
589 732 buildInputs = with self; [];
590 733 doCheck = false;
591 734 propagatedBuildInputs = with self; [];
592 735 src = fetchurl {
593 736 url = "https://pypi.python.org/packages/86/20/1948e0dfc4930ddde3da8c33612f6a5717c0b4bc28f591a5c5cf014dd390/venusian-1.0.tar.gz";
594 737 md5 = "dccf2eafb7113759d60c86faf5538756";
595 738 };
596 739 meta = {
597 740 license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
598 741 };
599 742 };
600 743 waitress = super.buildPythonPackage {
601 name = "waitress-0.8.9";
744 name = "waitress-1.0.1";
602 745 buildInputs = with self; [];
603 746 doCheck = false;
604 propagatedBuildInputs = with self; [setuptools];
747 propagatedBuildInputs = with self; [];
605 748 src = fetchurl {
606 url = "https://pypi.python.org/packages/ee/65/fc9dee74a909a1187ca51e4f15ad9c4d35476e4ab5813f73421505c48053/waitress-0.8.9.tar.gz";
607 md5 = "da3f2e62b3676be5dd630703a68e2a04";
749 url = "https://pypi.python.org/packages/78/7d/84d11b96c3f60164dec3bef4a859a03aeae0231aa93f57fbe0d05fa4ff36/waitress-1.0.1.tar.gz";
750 md5 = "dda92358a7569669086155923a46e57c";
608 751 };
609 752 meta = {
610 753 license = [ pkgs.lib.licenses.zpt21 ];
611 754 };
612 755 };
613 756 wcwidth = super.buildPythonPackage {
614 757 name = "wcwidth-0.1.7";
615 758 buildInputs = with self; [];
616 759 doCheck = false;
617 760 propagatedBuildInputs = with self; [];
618 761 src = fetchurl {
619 762 url = "https://pypi.python.org/packages/55/11/e4a2bb08bb450fdbd42cc709dd40de4ed2c472cf0ccb9e64af22279c5495/wcwidth-0.1.7.tar.gz";
620 763 md5 = "b3b6a0a08f0c8a34d1de8cf44150a4ad";
621 764 };
622 765 meta = {
623 766 license = [ pkgs.lib.licenses.mit ];
624 767 };
625 768 };
626 769 wheel = super.buildPythonPackage {
627 770 name = "wheel-0.29.0";
628 771 buildInputs = with self; [];
629 772 doCheck = false;
630 773 propagatedBuildInputs = with self; [];
631 774 src = fetchurl {
632 775 url = "https://pypi.python.org/packages/c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/wheel-0.29.0.tar.gz";
633 776 md5 = "555a67e4507cedee23a0deb9651e452f";
634 777 };
635 778 meta = {
636 779 license = [ pkgs.lib.licenses.mit ];
637 780 };
638 781 };
639 782 zope.deprecation = super.buildPythonPackage {
640 name = "zope.deprecation-4.1.1";
783 name = "zope.deprecation-4.1.2";
641 784 buildInputs = with self; [];
642 785 doCheck = false;
643 786 propagatedBuildInputs = with self; [setuptools];
644 787 src = fetchurl {
645 url = "https://pypi.python.org/packages/c5/c9/e760f131fcde817da6c186a3f4952b8f206b7eeb269bb6f0836c715c5f20/zope.deprecation-4.1.1.tar.gz";
646 md5 = "ce261b9384066f7e13b63525778430cb";
788 url = "https://pypi.python.org/packages/c1/d3/3919492d5e57d8dd01b36f30b34fc8404a30577392b1eb817c303499ad20/zope.deprecation-4.1.2.tar.gz";
789 md5 = "e9a663ded58f4f9f7881beb56cae2782";
647 790 };
648 791 meta = {
649 792 license = [ pkgs.lib.licenses.zpt21 ];
650 793 };
651 794 };
652 795 zope.interface = super.buildPythonPackage {
653 796 name = "zope.interface-4.1.3";
654 797 buildInputs = with self; [];
655 798 doCheck = false;
656 799 propagatedBuildInputs = with self; [setuptools];
657 800 src = fetchurl {
658 801 url = "https://pypi.python.org/packages/9d/81/2509ca3c6f59080123c1a8a97125eb48414022618cec0e64eb1313727bfe/zope.interface-4.1.3.tar.gz";
659 802 md5 = "9ae3d24c0c7415deb249dd1a132f0f79";
660 803 };
661 804 meta = {
662 805 license = [ pkgs.lib.licenses.zpt21 ];
663 806 };
664 807 };
665 808
666 809 ### Test requirements
667 810
668 pytest-sugar = super.buildPythonPackage {
669 name = "pytest-sugar-0.7.1";
670 buildInputs = with self; [];
671 doCheck = false;
672 propagatedBuildInputs = with self; [pytest termcolor];
673 src = fetchurl {
674 url = "https://pypi.python.org/packages/03/97/05d988b4fa870e7373e8ee4582408543b9ca2bd35c3c67b569369c6f9c49/pytest-sugar-0.7.1.tar.gz";
675 md5 = "7400f7c11f3d572b2c2a3b60352d35fe";
676 };
677 meta = {
678 license = [ pkgs.lib.licenses.bsdOriginal ];
679 };
680 };
681 termcolor = super.buildPythonPackage {
682 name = "termcolor-1.1.0";
683 buildInputs = with self; [];
684 doCheck = false;
685 propagatedBuildInputs = with self; [];
686 src = fetchurl {
687 url = "https://pypi.python.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz";
688 md5 = "043e89644f8909d462fbbfa511c768df";
689 };
690 meta = {
691 license = [ pkgs.lib.licenses.mit ];
692 };
693 };
811
694 812 }
@@ -1,35 +1,43 b''
1 # core
2 setuptools==30.1.0
3
1 4 Beaker==1.7.0
2 5 configobj==5.0.6
3 6 dulwich==0.13.0
4 7 hgsubversion==1.8.6
5 8 infrae.cache==1.0.1
6 ipdb==0.10.1
7 mercurial==3.8.4
8 msgpack-python==0.4.6
9 py==1.4.29
9 mercurial==4.0.2
10 msgpack-python==0.4.8
10 11 pyramid==1.6.1
11 12 pyramid-jinja2==2.5
12 13 pyramid-mako==1.0.2
13 Pyro4==4.41
14 pytest==2.8.5
15 14 repoze.lru==0.6
16 serpent==1.12
17 setuptools==20.8.1
18 15 simplejson==3.7.2
19 16 subprocess32==3.2.6
20 # TODO: johbo: This version is not in source on PyPI currently,
21 # change back once this or a future version is available
22 https://github.com/jelmer/subvertpy/archive/subvertpy-0.9.3.tar.gz#md5=7b745a47128050ea5a73efcd913ec1cf
17
18 # Custom subvertpy that is not available on pypi.
19 https://code.rhodecode.com/upstream/subvertpy/archive/subvertpy-0.9.3.tar.gz?md5=4e49da2fe07608239cc9a80a7bb8f33c#egg=subvertpy==0.9.3
20
23 21 six==1.9.0
24 22 translationstring==1.3
25 waitress==0.8.9
26 23 WebOb==1.3.1
27 24 wheel==0.29.0
28 zope.deprecation==4.1.1
25 zope.deprecation==4.1.2
29 26 zope.interface==4.1.3
30 greenlet==0.4.7
27
28 ## debug
29 ipdb==0.10.1
30 ipython==5.1.0
31
32 # http servers
33 gevent==1.1.2
34 greenlet==0.4.10
31 35 gunicorn==19.6.0
36 waitress==1.0.1
32 37
33 # Test related requirements
34 mock==1.0.1
35 WebTest==1.4.3
38 # Pyro/Deprecated TODO(Marcink): remove in 4.7 release.
39 Pyro4==4.41
40 serpent==1.15
41
42 ## test related requirements
43 -r requirements_test.txt
@@ -1,103 +1,132 b''
1 # -*- coding: utf-8 -*-
1 2 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
3 # Copyright (C) 2014-2017 RodeCode GmbH
3 4 #
4 5 # This program is free software; you can redistribute it and/or modify
5 6 # it under the terms of the GNU General Public License as published by
6 7 # the Free Software Foundation; either version 3 of the License, or
7 8 # (at your option) any later version.
8 9 #
9 10 # This program is distributed in the hope that it will be useful,
10 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 13 # GNU General Public License for more details.
13 14 #
14 15 # You should have received a copy of the GNU General Public License
15 16 # along with this program; if not, write to the Free Software Foundation,
16 17 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 18
19 # Import early to make sure things are patched up properly
18 20 from setuptools import setup, find_packages
19 from setuptools.command.test import test as TestCommand
21
22 import os
23 import sys
24 import pkgutil
25 import platform
26
27 from pip.download import PipSession
28 from pip.req import parse_requirements
29
20 30 from codecs import open
21 from os import path
22 import pkgutil
23 import sys
24 31
25 32
26 here = path.abspath(path.dirname(__file__))
33 if sys.version_info < (2, 7):
34 raise Exception('VCSServer requires Python 2.7 or later')
35
36 here = os.path.abspath(os.path.dirname(__file__))
37
38 # defines current platform
39 __platform__ = platform.system()
40 __license__ = 'GPL V3'
41 __author__ = 'RhodeCode GmbH'
42 __url__ = 'https://code.rhodecode.com'
43 is_windows = __platform__ in ('Windows',)
44
45
46 def _get_requirements(req_filename, exclude=None, extras=None):
47 extras = extras or []
48 exclude = exclude or []
27 49
28 with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
29 long_description = f.read()
50 try:
51 parsed = parse_requirements(
52 os.path.join(here, req_filename), session=PipSession())
53 except TypeError:
54 # try pip < 6.0.0, that doesn't support session
55 parsed = parse_requirements(os.path.join(here, req_filename))
56
57 requirements = []
58 for ir in parsed:
59 if ir.req and ir.name not in exclude:
60 requirements.append(str(ir.req))
61 return requirements + extras
62
63
64 # requirements extract
65 setup_requirements = ['pytest-runner']
66 install_requirements = _get_requirements(
67 'requirements.txt', exclude=['setuptools'])
68 test_requirements = _get_requirements(
69 'requirements_test.txt', extras=['configobj'])
30 70
31 71
32 72 def get_version():
33 73 version = pkgutil.get_data('vcsserver', 'VERSION')
34 74 return version.strip()
35 75
36 76
37 class PyTest(TestCommand):
38 user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")]
77 # additional files that goes into package itself
78 package_data = {
79 '': ['*.txt', '*.rst'],
80 'configs': ['*.ini'],
81 'vcsserver': ['VERSION'],
82 }
39 83
40 def initialize_options(self):
41 TestCommand.initialize_options(self)
42 self.pytest_args = []
84 description = 'Version Control System Server'
85 keywords = ' '.join([
86 'CLI', 'RhodeCode', 'RhodeCode Enterprise', 'RhodeCode Tools'])
43 87
44 def finalize_options(self):
45 TestCommand.finalize_options(self)
46 self.test_args = []
47 self.test_suite = True
48
49 def run_tests(self):
50 # import here, cause outside the eggs aren't loaded
51 import pytest
52 errno = pytest.main(self.pytest_args)
53 sys.exit(errno)
88 # README/DESCRIPTION generation
89 readme_file = 'README.rst'
90 changelog_file = 'CHANGES.rst'
91 try:
92 long_description = open(readme_file).read() + '\n\n' + \
93 open(changelog_file).read()
94 except IOError as err:
95 sys.stderr.write(
96 "[WARNING] Cannot find file specified as long_description (%s)\n "
97 "or changelog (%s) skipping that file" % (readme_file, changelog_file))
98 long_description = description
54 99
55 100
56 101 setup(
57 102 name='rhodecode-vcsserver',
58 103 version=get_version(),
59 description='Version Control System Server',
104 description=description,
60 105 long_description=long_description,
61 url='http://www.rhodecode.com',
62 author='RhodeCode GmbH',
106 keywords=keywords,
107 license=__license__,
108 author=__author__,
63 109 author_email='marcin@rhodecode.com',
64 cmdclass={'test': PyTest},
65 license='GPLv3',
110 url=__url__,
111 setup_requires=setup_requirements,
112 install_requires=install_requirements,
113 tests_require=test_requirements,
114 zip_safe=False,
115 packages=find_packages(exclude=["docs", "tests*"]),
116 package_data=package_data,
117 include_package_data=True,
66 118 classifiers=[
67 'Development Status :: 5 - Production/Stable',
119 'Development Status :: 6 - Mature',
68 120 'Intended Audience :: Developers',
121 'Operating System :: OS Independent',
69 122 'Topic :: Software Development :: Version Control',
70 123 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',
71 124 'Programming Language :: Python :: 2.7',
72 125 ],
73 packages=find_packages(),
74 tests_require=[
75 'mock',
76 'pytest',
77 'pytest-sugar',
78 'WebTest',
79 ],
80 install_requires=[
81 'configobj',
82 'dulwich',
83 'hgsubversion',
84 'infrae.cache',
85 'mercurial',
86 'msgpack-python',
87 'pyramid',
88 'Pyro4',
89 'simplejson',
90 'subprocess32',
91 'waitress',
92 'WebOb',
93 ],
94 package_data={
95 'vcsserver': ['VERSION'],
96 },
97 126 entry_points={
98 127 'console_scripts': [
99 128 'vcsserver=vcsserver.main:main',
100 129 ],
101 130 'paste.app_factory': ['main=vcsserver.http_main:main']
102 131 },
103 132 )
@@ -1,1 +1,1 b''
1 4.5.2 No newline at end of file
1 4.6.0 No newline at end of file
@@ -1,21 +1,21 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import pkgutil
19 19
20 20
21 21 __version__ = pkgutil.get_data('vcsserver', 'VERSION').strip()
@@ -1,82 +1,82 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import logging
19 19 import urlparse
20 20
21 21 log = logging.getLogger(__name__)
22 22
23 23
24 24 class RepoFactory(object):
25 25 """
26 26 Utility to create instances of repository
27 27
28 28 It provides internal caching of the `repo` object based on
29 29 the :term:`call context`.
30 30 """
31 31
32 32 def __init__(self, repo_cache):
33 33 self._cache = repo_cache
34 34
35 35 def _create_config(self, path, config):
36 36 config = {}
37 37 return config
38 38
39 39 def _create_repo(self, wire, create):
40 40 raise NotImplementedError()
41 41
42 42 def repo(self, wire, create=False):
43 43 """
44 44 Get a repository instance for the given path.
45 45
46 46 Uses internally the low level beaker API since the decorators introduce
47 47 significant overhead.
48 48 """
49 49 def create_new_repo():
50 50 return self._create_repo(wire, create)
51 51
52 52 return self._repo(wire, create_new_repo)
53 53
54 54 def _repo(self, wire, createfunc):
55 55 context = wire.get('context', None)
56 56 cache = wire.get('cache', True)
57 57
58 58 if context and cache:
59 59 cache_key = (context, wire['path'])
60 60 log.debug(
61 61 'FETCH %s@%s repo object from cache. Context: %s',
62 62 self.__class__.__name__, wire['path'], context)
63 63 return self._cache.get(key=cache_key, createfunc=createfunc)
64 64 else:
65 65 log.debug(
66 66 'INIT %s@%s repo object based on wire %s. Context: %s',
67 67 self.__class__.__name__, wire['path'], wire, context)
68 68 return createfunc()
69 69
70 70
71 71 def obfuscate_qs(query_string):
72 72 if query_string is None:
73 73 return None
74 74
75 75 parsed = []
76 76 for k, v in urlparse.parse_qsl(query_string, keep_blank_values=True):
77 77 if k in ['auth_token', 'api_key']:
78 78 v = "*****"
79 79 parsed.append((k, v))
80 80
81 81 return '&'.join('{}{}'.format(
82 82 k, '={}'.format(v) if v else '') for k, v in parsed)
@@ -1,70 +1,70 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 """
19 19 Special exception handling over the wire.
20 20
21 21 Since we cannot assume that our client is able to import our exception classes,
22 22 this module provides a "wrapping" mechanism to raise plain exceptions
23 23 which contain an extra attribute `_vcs_kind` to allow a client to distinguish
24 24 different error conditions.
25 25 """
26 26
27 27 import functools
28 28 from pyramid.httpexceptions import HTTPLocked
29 29
30 30
31 31 def _make_exception(kind, *args):
32 32 """
33 33 Prepares a base `Exception` instance to be sent over the wire.
34 34
35 35 To give our caller a hint what this is about, it will attach an attribute
36 36 `_vcs_kind` to the exception.
37 37 """
38 38 exc = Exception(*args)
39 39 exc._vcs_kind = kind
40 40 return exc
41 41
42 42
43 43 AbortException = functools.partial(_make_exception, 'abort')
44 44
45 45 ArchiveException = functools.partial(_make_exception, 'archive')
46 46
47 47 LookupException = functools.partial(_make_exception, 'lookup')
48 48
49 49 VcsException = functools.partial(_make_exception, 'error')
50 50
51 51 RepositoryLockedException = functools.partial(_make_exception, 'repo_locked')
52 52
53 53 RequirementException = functools.partial(_make_exception, 'requirement')
54 54
55 55 UnhandledException = functools.partial(_make_exception, 'unhandled')
56 56
57 57 URLError = functools.partial(_make_exception, 'url_error')
58 58
59 59 SubrepoMergeException = functools.partial(_make_exception, 'subrepo_merge_error')
60 60
61 61
62 62 class HTTPRepoLocked(HTTPLocked):
63 63 """
64 64 Subclass of HTTPLocked response that allows to set the title and status
65 65 code via constructor arguments.
66 66 """
67 67 def __init__(self, title, status_code=None, **kwargs):
68 68 self.code = status_code or HTTPLocked.code
69 69 self.title = title
70 70 super(HTTPRepoLocked, self).__init__(**kwargs)
@@ -1,581 +1,581 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import logging
19 19 import os
20 20 import posixpath as vcspath
21 21 import re
22 22 import stat
23 23 import urllib
24 24 import urllib2
25 25 from functools import wraps
26 26
27 27 from dulwich import index, objects
28 28 from dulwich.client import HttpGitClient, LocalGitClient
29 29 from dulwich.errors import (
30 30 NotGitRepository, ChecksumMismatch, WrongObjectException,
31 31 MissingCommitError, ObjectMissing, HangupException,
32 32 UnexpectedCommandError)
33 33 from dulwich.repo import Repo as DulwichRepo, Tag
34 34 from dulwich.server import update_server_info
35 35
36 36 from vcsserver import exceptions, settings, subprocessio
37 37 from vcsserver.utils import safe_str
38 38 from vcsserver.base import RepoFactory, obfuscate_qs
39 39 from vcsserver.hgcompat import (
40 40 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
41 41
42 42
43 43 DIR_STAT = stat.S_IFDIR
44 44 FILE_MODE = stat.S_IFMT
45 45 GIT_LINK = objects.S_IFGITLINK
46 46
47 47 log = logging.getLogger(__name__)
48 48
49 49
50 50 def reraise_safe_exceptions(func):
51 51 """Converts Dulwich exceptions to something neutral."""
52 52 @wraps(func)
53 53 def wrapper(*args, **kwargs):
54 54 try:
55 55 return func(*args, **kwargs)
56 56 except (ChecksumMismatch, WrongObjectException, MissingCommitError,
57 57 ObjectMissing) as e:
58 58 raise exceptions.LookupException(e.message)
59 59 except (HangupException, UnexpectedCommandError) as e:
60 60 raise exceptions.VcsException(e.message)
61 61 return wrapper
62 62
63 63
64 64 class Repo(DulwichRepo):
65 65 """
66 66 A wrapper for dulwich Repo class.
67 67
68 68 Since dulwich is sometimes keeping .idx file descriptors open, it leads to
69 69 "Too many open files" error. We need to close all opened file descriptors
70 70 once the repo object is destroyed.
71 71
72 72 TODO: mikhail: please check if we need this wrapper after updating dulwich
73 73 to 0.12.0 +
74 74 """
75 75 def __del__(self):
76 76 if hasattr(self, 'object_store'):
77 77 self.close()
78 78
79 79
80 80 class GitFactory(RepoFactory):
81 81
82 82 def _create_repo(self, wire, create):
83 83 repo_path = str_to_dulwich(wire['path'])
84 84 return Repo(repo_path)
85 85
86 86
87 87 class GitRemote(object):
88 88
89 89 def __init__(self, factory):
90 90 self._factory = factory
91 91
92 92 self._bulk_methods = {
93 93 "author": self.commit_attribute,
94 94 "date": self.get_object_attrs,
95 95 "message": self.commit_attribute,
96 96 "parents": self.commit_attribute,
97 97 "_commit": self.revision,
98 98 }
99 99
100 100 def _assign_ref(self, wire, ref, commit_id):
101 101 repo = self._factory.repo(wire)
102 102 repo[ref] = commit_id
103 103
104 104 @reraise_safe_exceptions
105 105 def add_object(self, wire, content):
106 106 repo = self._factory.repo(wire)
107 107 blob = objects.Blob()
108 108 blob.set_raw_string(content)
109 109 repo.object_store.add_object(blob)
110 110 return blob.id
111 111
112 112 @reraise_safe_exceptions
113 113 def assert_correct_path(self, wire):
114 114 try:
115 115 self._factory.repo(wire)
116 116 except NotGitRepository as e:
117 117 # Exception can contain unicode which we convert
118 118 raise exceptions.AbortException(repr(e))
119 119
120 120 @reraise_safe_exceptions
121 121 def bare(self, wire):
122 122 repo = self._factory.repo(wire)
123 123 return repo.bare
124 124
125 125 @reraise_safe_exceptions
126 126 def blob_as_pretty_string(self, wire, sha):
127 127 repo = self._factory.repo(wire)
128 128 return repo[sha].as_pretty_string()
129 129
130 130 @reraise_safe_exceptions
131 131 def blob_raw_length(self, wire, sha):
132 132 repo = self._factory.repo(wire)
133 133 blob = repo[sha]
134 134 return blob.raw_length()
135 135
136 136 @reraise_safe_exceptions
137 137 def bulk_request(self, wire, rev, pre_load):
138 138 result = {}
139 139 for attr in pre_load:
140 140 try:
141 141 method = self._bulk_methods[attr]
142 142 args = [wire, rev]
143 143 if attr == "date":
144 144 args.extend(["commit_time", "commit_timezone"])
145 145 elif attr in ["author", "message", "parents"]:
146 146 args.append(attr)
147 147 result[attr] = method(*args)
148 148 except KeyError:
149 149 raise exceptions.VcsException(
150 150 "Unknown bulk attribute: %s" % attr)
151 151 return result
152 152
153 153 def _build_opener(self, url):
154 154 handlers = []
155 155 url_obj = url_parser(url)
156 156 _, authinfo = url_obj.authinfo()
157 157
158 158 if authinfo:
159 159 # create a password manager
160 160 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
161 161 passmgr.add_password(*authinfo)
162 162
163 163 handlers.extend((httpbasicauthhandler(passmgr),
164 164 httpdigestauthhandler(passmgr)))
165 165
166 166 return urllib2.build_opener(*handlers)
167 167
168 168 @reraise_safe_exceptions
169 169 def check_url(self, url, config):
170 170 url_obj = url_parser(url)
171 171 test_uri, _ = url_obj.authinfo()
172 172 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
173 173 url_obj.query = obfuscate_qs(url_obj.query)
174 174 cleaned_uri = str(url_obj)
175 175 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
176 176
177 177 if not test_uri.endswith('info/refs'):
178 178 test_uri = test_uri.rstrip('/') + '/info/refs'
179 179
180 180 o = self._build_opener(url)
181 181 o.addheaders = [('User-Agent', 'git/1.7.8.0')] # fake some git
182 182
183 183 q = {"service": 'git-upload-pack'}
184 184 qs = '?%s' % urllib.urlencode(q)
185 185 cu = "%s%s" % (test_uri, qs)
186 186 req = urllib2.Request(cu, None, {})
187 187
188 188 try:
189 189 log.debug("Trying to open URL %s", cleaned_uri)
190 190 resp = o.open(req)
191 191 if resp.code != 200:
192 192 raise exceptions.URLError('Return Code is not 200')
193 193 except Exception as e:
194 194 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
195 195 # means it cannot be cloned
196 196 raise exceptions.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
197 197
198 198 # now detect if it's proper git repo
199 199 gitdata = resp.read()
200 200 if 'service=git-upload-pack' in gitdata:
201 201 pass
202 202 elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
203 203 # old style git can return some other format !
204 204 pass
205 205 else:
206 206 raise exceptions.URLError(
207 207 "url [%s] does not look like an git" % (cleaned_uri,))
208 208
209 209 return True
210 210
211 211 @reraise_safe_exceptions
212 212 def clone(self, wire, url, deferred, valid_refs, update_after_clone):
213 213 remote_refs = self.fetch(wire, url, apply_refs=False)
214 214 repo = self._factory.repo(wire)
215 215 if isinstance(valid_refs, list):
216 216 valid_refs = tuple(valid_refs)
217 217
218 218 for k in remote_refs:
219 219 # only parse heads/tags and skip so called deferred tags
220 220 if k.startswith(valid_refs) and not k.endswith(deferred):
221 221 repo[k] = remote_refs[k]
222 222
223 223 if update_after_clone:
224 224 # we want to checkout HEAD
225 225 repo["HEAD"] = remote_refs["HEAD"]
226 226 index.build_index_from_tree(repo.path, repo.index_path(),
227 227 repo.object_store, repo["HEAD"].tree)
228 228
229 229 # TODO: this is quite complex, check if that can be simplified
230 230 @reraise_safe_exceptions
231 231 def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
232 232 repo = self._factory.repo(wire)
233 233 object_store = repo.object_store
234 234
235 235 # Create tree and populates it with blobs
236 236 commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
237 237
238 238 for node in updated:
239 239 # Compute subdirs if needed
240 240 dirpath, nodename = vcspath.split(node['path'])
241 241 dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
242 242 parent = commit_tree
243 243 ancestors = [('', parent)]
244 244
245 245 # Tries to dig for the deepest existing tree
246 246 while dirnames:
247 247 curdir = dirnames.pop(0)
248 248 try:
249 249 dir_id = parent[curdir][1]
250 250 except KeyError:
251 251 # put curdir back into dirnames and stops
252 252 dirnames.insert(0, curdir)
253 253 break
254 254 else:
255 255 # If found, updates parent
256 256 parent = repo[dir_id]
257 257 ancestors.append((curdir, parent))
258 258 # Now parent is deepest existing tree and we need to create
259 259 # subtrees for dirnames (in reverse order)
260 260 # [this only applies for nodes from added]
261 261 new_trees = []
262 262
263 263 blob = objects.Blob.from_string(node['content'])
264 264
265 265 if dirnames:
266 266 # If there are trees which should be created we need to build
267 267 # them now (in reverse order)
268 268 reversed_dirnames = list(reversed(dirnames))
269 269 curtree = objects.Tree()
270 270 curtree[node['node_path']] = node['mode'], blob.id
271 271 new_trees.append(curtree)
272 272 for dirname in reversed_dirnames[:-1]:
273 273 newtree = objects.Tree()
274 274 newtree[dirname] = (DIR_STAT, curtree.id)
275 275 new_trees.append(newtree)
276 276 curtree = newtree
277 277 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
278 278 else:
279 279 parent.add(
280 280 name=node['node_path'], mode=node['mode'], hexsha=blob.id)
281 281
282 282 new_trees.append(parent)
283 283 # Update ancestors
284 284 reversed_ancestors = reversed(
285 285 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
286 286 for parent, tree, path in reversed_ancestors:
287 287 parent[path] = (DIR_STAT, tree.id)
288 288 object_store.add_object(tree)
289 289
290 290 object_store.add_object(blob)
291 291 for tree in new_trees:
292 292 object_store.add_object(tree)
293 293
294 294 for node_path in removed:
295 295 paths = node_path.split('/')
296 296 tree = commit_tree
297 297 trees = [tree]
298 298 # Traverse deep into the forest...
299 299 for path in paths:
300 300 try:
301 301 obj = repo[tree[path][1]]
302 302 if isinstance(obj, objects.Tree):
303 303 trees.append(obj)
304 304 tree = obj
305 305 except KeyError:
306 306 break
307 307 # Cut down the blob and all rotten trees on the way back...
308 308 for path, tree in reversed(zip(paths, trees)):
309 309 del tree[path]
310 310 if tree:
311 311 # This tree still has elements - don't remove it or any
312 312 # of it's parents
313 313 break
314 314
315 315 object_store.add_object(commit_tree)
316 316
317 317 # Create commit
318 318 commit = objects.Commit()
319 319 commit.tree = commit_tree.id
320 320 for k, v in commit_data.iteritems():
321 321 setattr(commit, k, v)
322 322 object_store.add_object(commit)
323 323
324 324 ref = 'refs/heads/%s' % branch
325 325 repo.refs[ref] = commit.id
326 326
327 327 return commit.id
328 328
329 329 @reraise_safe_exceptions
330 330 def fetch(self, wire, url, apply_refs=True, refs=None):
331 331 if url != 'default' and '://' not in url:
332 332 client = LocalGitClient(url)
333 333 else:
334 334 url_obj = url_parser(url)
335 335 o = self._build_opener(url)
336 336 url, _ = url_obj.authinfo()
337 337 client = HttpGitClient(base_url=url, opener=o)
338 338 repo = self._factory.repo(wire)
339 339
340 340 determine_wants = repo.object_store.determine_wants_all
341 341 if refs:
342 342 def determine_wants_requested(references):
343 343 return [references[r] for r in references if r in refs]
344 344 determine_wants = determine_wants_requested
345 345
346 346 try:
347 347 remote_refs = client.fetch(
348 348 path=url, target=repo, determine_wants=determine_wants)
349 349 except NotGitRepository as e:
350 350 log.warning(
351 351 'Trying to fetch from "%s" failed, not a Git repository.', url)
352 352 # Exception can contain unicode which we convert
353 353 raise exceptions.AbortException(repr(e))
354 354
355 355 # mikhail: client.fetch() returns all the remote refs, but fetches only
356 356 # refs filtered by `determine_wants` function. We need to filter result
357 357 # as well
358 358 if refs:
359 359 remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
360 360
361 361 if apply_refs:
362 362 # TODO: johbo: Needs proper test coverage with a git repository
363 363 # that contains a tag object, so that we would end up with
364 364 # a peeled ref at this point.
365 365 PEELED_REF_MARKER = '^{}'
366 366 for k in remote_refs:
367 367 if k.endswith(PEELED_REF_MARKER):
368 368 log.info("Skipping peeled reference %s", k)
369 369 continue
370 370 repo[k] = remote_refs[k]
371 371
372 372 if refs:
373 373 # mikhail: explicitly set the head to the last ref.
374 374 repo['HEAD'] = remote_refs[refs[-1]]
375 375
376 376 # TODO: mikhail: should we return remote_refs here to be
377 377 # consistent?
378 378 else:
379 379 return remote_refs
380 380
381 381 @reraise_safe_exceptions
382 382 def get_remote_refs(self, wire, url):
383 383 repo = Repo(url)
384 384 return repo.get_refs()
385 385
386 386 @reraise_safe_exceptions
387 387 def get_description(self, wire):
388 388 repo = self._factory.repo(wire)
389 389 return repo.get_description()
390 390
391 391 @reraise_safe_exceptions
392 392 def get_file_history(self, wire, file_path, commit_id, limit):
393 393 repo = self._factory.repo(wire)
394 394 include = [commit_id]
395 395 paths = [file_path]
396 396
397 397 walker = repo.get_walker(include, paths=paths, max_entries=limit)
398 398 return [x.commit.id for x in walker]
399 399
400 400 @reraise_safe_exceptions
401 401 def get_missing_revs(self, wire, rev1, rev2, path2):
402 402 repo = self._factory.repo(wire)
403 403 LocalGitClient(thin_packs=False).fetch(path2, repo)
404 404
405 405 wire_remote = wire.copy()
406 406 wire_remote['path'] = path2
407 407 repo_remote = self._factory.repo(wire_remote)
408 408 LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
409 409
410 410 revs = [
411 411 x.commit.id
412 412 for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
413 413 return revs
414 414
415 415 @reraise_safe_exceptions
416 416 def get_object(self, wire, sha):
417 417 repo = self._factory.repo(wire)
418 418 obj = repo.get_object(sha)
419 419 commit_id = obj.id
420 420
421 421 if isinstance(obj, Tag):
422 422 commit_id = obj.object[1]
423 423
424 424 return {
425 425 'id': obj.id,
426 426 'type': obj.type_name,
427 427 'commit_id': commit_id
428 428 }
429 429
430 430 @reraise_safe_exceptions
431 431 def get_object_attrs(self, wire, sha, *attrs):
432 432 repo = self._factory.repo(wire)
433 433 obj = repo.get_object(sha)
434 434 return list(getattr(obj, a) for a in attrs)
435 435
436 436 @reraise_safe_exceptions
437 437 def get_refs(self, wire):
438 438 repo = self._factory.repo(wire)
439 439 result = {}
440 440 for ref, sha in repo.refs.as_dict().items():
441 441 peeled_sha = repo.get_peeled(ref)
442 442 result[ref] = peeled_sha
443 443 return result
444 444
445 445 @reraise_safe_exceptions
446 446 def get_refs_path(self, wire):
447 447 repo = self._factory.repo(wire)
448 448 return repo.refs.path
449 449
450 450 @reraise_safe_exceptions
451 451 def head(self, wire):
452 452 repo = self._factory.repo(wire)
453 453 return repo.head()
454 454
455 455 @reraise_safe_exceptions
456 456 def init(self, wire):
457 457 repo_path = str_to_dulwich(wire['path'])
458 458 self.repo = Repo.init(repo_path)
459 459
460 460 @reraise_safe_exceptions
461 461 def init_bare(self, wire):
462 462 repo_path = str_to_dulwich(wire['path'])
463 463 self.repo = Repo.init_bare(repo_path)
464 464
465 465 @reraise_safe_exceptions
466 466 def revision(self, wire, rev):
467 467 repo = self._factory.repo(wire)
468 468 obj = repo[rev]
469 469 obj_data = {
470 470 'id': obj.id,
471 471 }
472 472 try:
473 473 obj_data['tree'] = obj.tree
474 474 except AttributeError:
475 475 pass
476 476 return obj_data
477 477
478 478 @reraise_safe_exceptions
479 479 def commit_attribute(self, wire, rev, attr):
480 480 repo = self._factory.repo(wire)
481 481 obj = repo[rev]
482 482 return getattr(obj, attr)
483 483
484 484 @reraise_safe_exceptions
485 485 def set_refs(self, wire, key, value):
486 486 repo = self._factory.repo(wire)
487 487 repo.refs[key] = value
488 488
489 489 @reraise_safe_exceptions
490 490 def remove_ref(self, wire, key):
491 491 repo = self._factory.repo(wire)
492 492 del repo.refs[key]
493 493
494 494 @reraise_safe_exceptions
495 495 def tree_changes(self, wire, source_id, target_id):
496 496 repo = self._factory.repo(wire)
497 497 source = repo[source_id].tree if source_id else None
498 498 target = repo[target_id].tree
499 499 result = repo.object_store.tree_changes(source, target)
500 500 return list(result)
501 501
502 502 @reraise_safe_exceptions
503 503 def tree_items(self, wire, tree_id):
504 504 repo = self._factory.repo(wire)
505 505 tree = repo[tree_id]
506 506
507 507 result = []
508 508 for item in tree.iteritems():
509 509 item_sha = item.sha
510 510 item_mode = item.mode
511 511
512 512 if FILE_MODE(item_mode) == GIT_LINK:
513 513 item_type = "link"
514 514 else:
515 515 item_type = repo[item_sha].type_name
516 516
517 517 result.append((item.path, item_mode, item_sha, item_type))
518 518 return result
519 519
520 520 @reraise_safe_exceptions
521 521 def update_server_info(self, wire):
522 522 repo = self._factory.repo(wire)
523 523 update_server_info(repo)
524 524
525 525 @reraise_safe_exceptions
526 526 def discover_git_version(self):
527 527 stdout, _ = self.run_git_command(
528 528 {}, ['--version'], _bare=True, _safe=True)
529 529 prefix = 'git version'
530 530 if stdout.startswith(prefix):
531 531 stdout = stdout[len(prefix):]
532 532 return stdout.strip()
533 533
534 534 @reraise_safe_exceptions
535 535 def run_git_command(self, wire, cmd, **opts):
536 536 path = wire.get('path', None)
537 537
538 538 if path and os.path.isdir(path):
539 539 opts['cwd'] = path
540 540
541 541 if '_bare' in opts:
542 542 _copts = []
543 543 del opts['_bare']
544 544 else:
545 545 _copts = ['-c', 'core.quotepath=false', ]
546 546 safe_call = False
547 547 if '_safe' in opts:
548 548 # no exc on failure
549 549 del opts['_safe']
550 550 safe_call = True
551 551
552 552 gitenv = os.environ.copy()
553 553 gitenv.update(opts.pop('extra_env', {}))
554 554 # need to clean fix GIT_DIR !
555 555 if 'GIT_DIR' in gitenv:
556 556 del gitenv['GIT_DIR']
557 557 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
558 558
559 559 cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
560 560
561 561 try:
562 562 _opts = {'env': gitenv, 'shell': False}
563 563 _opts.update(opts)
564 564 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
565 565
566 566 return ''.join(p), ''.join(p.error)
567 567 except (EnvironmentError, OSError) as err:
568 568 tb_err = ("Couldn't run git command (%s).\n"
569 569 "Original error was:%s\n" % (cmd, err))
570 570 log.exception(tb_err)
571 571 if safe_call:
572 572 return '', err
573 573 else:
574 574 raise exceptions.VcsException(tb_err)
575 575
576 576
577 577 def str_to_dulwich(value):
578 578 """
579 579 Dulwich 0.10.1a requires `unicode` objects to be passed in.
580 580 """
581 581 return value.decode(settings.WIRE_ENCODING)
@@ -1,725 +1,725 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import io
19 19 import logging
20 20 import stat
21 21 import sys
22 22 import urllib
23 23 import urllib2
24 24
25 25 from hgext import largefiles, rebase
26 26 from hgext.strip import strip as hgext_strip
27 27 from mercurial import commands
28 28 from mercurial import unionrepo
29 29
30 30 from vcsserver import exceptions
31 31 from vcsserver.base import RepoFactory, obfuscate_qs
32 32 from vcsserver.hgcompat import (
33 33 archival, bin, clone, config as hgconfig, diffopts, hex,
34 34 hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
35 35 httppeer, localrepository, match, memctx, exchange, memfilectx, nullrev,
36 36 patch, peer, revrange, ui, Abort, LookupError, RepoError, RepoLookupError,
37 37 InterventionRequired, RequirementError)
38 38
39 39 log = logging.getLogger(__name__)
40 40
41 41
42 42 def make_ui_from_config(repo_config):
43 43 baseui = ui.ui()
44 44
45 45 # clean the baseui object
46 46 baseui._ocfg = hgconfig.config()
47 47 baseui._ucfg = hgconfig.config()
48 48 baseui._tcfg = hgconfig.config()
49 49
50 50 for section, option, value in repo_config:
51 51 baseui.setconfig(section, option, value)
52 52
53 53 # make our hgweb quiet so it doesn't print output
54 54 baseui.setconfig('ui', 'quiet', 'true')
55 55
56 56 # force mercurial to only use 1 thread, otherwise it may try to set a
57 57 # signal in a non-main thread, thus generating a ValueError.
58 58 baseui.setconfig('worker', 'numcpus', 1)
59 59
60 60 # If there is no config for the largefiles extension, we explicitly disable
61 61 # it here. This overrides settings from repositories hgrc file. Recent
62 62 # mercurial versions enable largefiles in hgrc on clone from largefile
63 63 # repo.
64 64 if not baseui.hasconfig('extensions', 'largefiles'):
65 65 log.debug('Explicitly disable largefiles extension for repo.')
66 66 baseui.setconfig('extensions', 'largefiles', '!')
67 67
68 68 return baseui
69 69
70 70
71 71 def reraise_safe_exceptions(func):
72 72 """Decorator for converting mercurial exceptions to something neutral."""
73 73 def wrapper(*args, **kwargs):
74 74 try:
75 75 return func(*args, **kwargs)
76 76 except (Abort, InterventionRequired):
77 77 raise_from_original(exceptions.AbortException)
78 78 except RepoLookupError:
79 79 raise_from_original(exceptions.LookupException)
80 80 except RequirementError:
81 81 raise_from_original(exceptions.RequirementException)
82 82 except RepoError:
83 83 raise_from_original(exceptions.VcsException)
84 84 except LookupError:
85 85 raise_from_original(exceptions.LookupException)
86 86 except Exception as e:
87 87 if not hasattr(e, '_vcs_kind'):
88 88 log.exception("Unhandled exception in hg remote call")
89 89 raise_from_original(exceptions.UnhandledException)
90 90 raise
91 91 return wrapper
92 92
93 93
94 94 def raise_from_original(new_type):
95 95 """
96 96 Raise a new exception type with original args and traceback.
97 97 """
98 98 _, original, traceback = sys.exc_info()
99 99 try:
100 100 raise new_type(*original.args), None, traceback
101 101 finally:
102 102 del traceback
103 103
104 104
105 105 class MercurialFactory(RepoFactory):
106 106
107 107 def _create_config(self, config, hooks=True):
108 108 if not hooks:
109 109 hooks_to_clean = frozenset((
110 110 'changegroup.repo_size', 'preoutgoing.pre_pull',
111 111 'outgoing.pull_logger', 'prechangegroup.pre_push'))
112 112 new_config = []
113 113 for section, option, value in config:
114 114 if section == 'hooks' and option in hooks_to_clean:
115 115 continue
116 116 new_config.append((section, option, value))
117 117 config = new_config
118 118
119 119 baseui = make_ui_from_config(config)
120 120 return baseui
121 121
122 122 def _create_repo(self, wire, create):
123 123 baseui = self._create_config(wire["config"])
124 124 return localrepository(baseui, wire["path"], create)
125 125
126 126
127 127 class HgRemote(object):
128 128
129 129 def __init__(self, factory):
130 130 self._factory = factory
131 131
132 132 self._bulk_methods = {
133 133 "affected_files": self.ctx_files,
134 134 "author": self.ctx_user,
135 135 "branch": self.ctx_branch,
136 136 "children": self.ctx_children,
137 137 "date": self.ctx_date,
138 138 "message": self.ctx_description,
139 139 "parents": self.ctx_parents,
140 140 "status": self.ctx_status,
141 141 "_file_paths": self.ctx_list,
142 142 }
143 143
144 144 @reraise_safe_exceptions
145 145 def discover_hg_version(self):
146 146 from mercurial import util
147 147 return util.version()
148 148
149 149 @reraise_safe_exceptions
150 150 def archive_repo(self, archive_path, mtime, file_info, kind):
151 151 if kind == "tgz":
152 152 archiver = archival.tarit(archive_path, mtime, "gz")
153 153 elif kind == "tbz2":
154 154 archiver = archival.tarit(archive_path, mtime, "bz2")
155 155 elif kind == 'zip':
156 156 archiver = archival.zipit(archive_path, mtime)
157 157 else:
158 158 raise exceptions.ArchiveException(
159 159 'Remote does not support: "%s".' % kind)
160 160
161 161 for f_path, f_mode, f_is_link, f_content in file_info:
162 162 archiver.addfile(f_path, f_mode, f_is_link, f_content)
163 163 archiver.done()
164 164
165 165 @reraise_safe_exceptions
166 166 def bookmarks(self, wire):
167 167 repo = self._factory.repo(wire)
168 168 return dict(repo._bookmarks)
169 169
170 170 @reraise_safe_exceptions
171 171 def branches(self, wire, normal, closed):
172 172 repo = self._factory.repo(wire)
173 173 iter_branches = repo.branchmap().iterbranches()
174 174 bt = {}
175 175 for branch_name, _heads, tip, is_closed in iter_branches:
176 176 if normal and not is_closed:
177 177 bt[branch_name] = tip
178 178 if closed and is_closed:
179 179 bt[branch_name] = tip
180 180
181 181 return bt
182 182
183 183 @reraise_safe_exceptions
184 184 def bulk_request(self, wire, rev, pre_load):
185 185 result = {}
186 186 for attr in pre_load:
187 187 try:
188 188 method = self._bulk_methods[attr]
189 189 result[attr] = method(wire, rev)
190 190 except KeyError:
191 191 raise exceptions.VcsException(
192 192 'Unknown bulk attribute: "%s"' % attr)
193 193 return result
194 194
195 195 @reraise_safe_exceptions
196 196 def clone(self, wire, source, dest, update_after_clone=False, hooks=True):
197 197 baseui = self._factory._create_config(wire["config"], hooks=hooks)
198 198 clone(baseui, source, dest, noupdate=not update_after_clone)
199 199
200 200 @reraise_safe_exceptions
201 201 def commitctx(
202 202 self, wire, message, parents, commit_time, commit_timezone,
203 203 user, files, extra, removed, updated):
204 204
205 205 def _filectxfn(_repo, memctx, path):
206 206 """
207 207 Marks given path as added/changed/removed in a given _repo. This is
208 208 for internal mercurial commit function.
209 209 """
210 210
211 211 # check if this path is removed
212 212 if path in removed:
213 213 # returning None is a way to mark node for removal
214 214 return None
215 215
216 216 # check if this path is added
217 217 for node in updated:
218 218 if node['path'] == path:
219 219 return memfilectx(
220 220 _repo,
221 221 path=node['path'],
222 222 data=node['content'],
223 223 islink=False,
224 224 isexec=bool(node['mode'] & stat.S_IXUSR),
225 225 copied=False,
226 226 memctx=memctx)
227 227
228 228 raise exceptions.AbortException(
229 229 "Given path haven't been marked as added, "
230 230 "changed or removed (%s)" % path)
231 231
232 232 repo = self._factory.repo(wire)
233 233
234 234 commit_ctx = memctx(
235 235 repo=repo,
236 236 parents=parents,
237 237 text=message,
238 238 files=files,
239 239 filectxfn=_filectxfn,
240 240 user=user,
241 241 date=(commit_time, commit_timezone),
242 242 extra=extra)
243 243
244 244 n = repo.commitctx(commit_ctx)
245 245 new_id = hex(n)
246 246
247 247 return new_id
248 248
249 249 @reraise_safe_exceptions
250 250 def ctx_branch(self, wire, revision):
251 251 repo = self._factory.repo(wire)
252 252 ctx = repo[revision]
253 253 return ctx.branch()
254 254
255 255 @reraise_safe_exceptions
256 256 def ctx_children(self, wire, revision):
257 257 repo = self._factory.repo(wire)
258 258 ctx = repo[revision]
259 259 return [child.rev() for child in ctx.children()]
260 260
261 261 @reraise_safe_exceptions
262 262 def ctx_date(self, wire, revision):
263 263 repo = self._factory.repo(wire)
264 264 ctx = repo[revision]
265 265 return ctx.date()
266 266
267 267 @reraise_safe_exceptions
268 268 def ctx_description(self, wire, revision):
269 269 repo = self._factory.repo(wire)
270 270 ctx = repo[revision]
271 271 return ctx.description()
272 272
273 273 @reraise_safe_exceptions
274 274 def ctx_diff(
275 275 self, wire, revision, git=True, ignore_whitespace=True, context=3):
276 276 repo = self._factory.repo(wire)
277 277 ctx = repo[revision]
278 278 result = ctx.diff(
279 279 git=git, ignore_whitespace=ignore_whitespace, context=context)
280 280 return list(result)
281 281
282 282 @reraise_safe_exceptions
283 283 def ctx_files(self, wire, revision):
284 284 repo = self._factory.repo(wire)
285 285 ctx = repo[revision]
286 286 return ctx.files()
287 287
288 288 @reraise_safe_exceptions
289 289 def ctx_list(self, path, revision):
290 290 repo = self._factory.repo(path)
291 291 ctx = repo[revision]
292 292 return list(ctx)
293 293
294 294 @reraise_safe_exceptions
295 295 def ctx_parents(self, wire, revision):
296 296 repo = self._factory.repo(wire)
297 297 ctx = repo[revision]
298 298 return [parent.rev() for parent in ctx.parents()]
299 299
300 300 @reraise_safe_exceptions
301 301 def ctx_substate(self, wire, revision):
302 302 repo = self._factory.repo(wire)
303 303 ctx = repo[revision]
304 304 return ctx.substate
305 305
306 306 @reraise_safe_exceptions
307 307 def ctx_status(self, wire, revision):
308 308 repo = self._factory.repo(wire)
309 309 ctx = repo[revision]
310 310 status = repo[ctx.p1().node()].status(other=ctx.node())
311 311 # object of status (odd, custom named tuple in mercurial) is not
312 312 # correctly serializable via Pyro, we make it a list, as the underling
313 313 # API expects this to be a list
314 314 return list(status)
315 315
316 316 @reraise_safe_exceptions
317 317 def ctx_user(self, wire, revision):
318 318 repo = self._factory.repo(wire)
319 319 ctx = repo[revision]
320 320 return ctx.user()
321 321
322 322 @reraise_safe_exceptions
323 323 def check_url(self, url, config):
324 324 _proto = None
325 325 if '+' in url[:url.find('://')]:
326 326 _proto = url[0:url.find('+')]
327 327 url = url[url.find('+') + 1:]
328 328 handlers = []
329 329 url_obj = url_parser(url)
330 330 test_uri, authinfo = url_obj.authinfo()
331 331 url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
332 332 url_obj.query = obfuscate_qs(url_obj.query)
333 333
334 334 cleaned_uri = str(url_obj)
335 335 log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
336 336
337 337 if authinfo:
338 338 # create a password manager
339 339 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
340 340 passmgr.add_password(*authinfo)
341 341
342 342 handlers.extend((httpbasicauthhandler(passmgr),
343 343 httpdigestauthhandler(passmgr)))
344 344
345 345 o = urllib2.build_opener(*handlers)
346 346 o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
347 347 ('Accept', 'application/mercurial-0.1')]
348 348
349 349 q = {"cmd": 'between'}
350 350 q.update({'pairs': "%s-%s" % ('0' * 40, '0' * 40)})
351 351 qs = '?%s' % urllib.urlencode(q)
352 352 cu = "%s%s" % (test_uri, qs)
353 353 req = urllib2.Request(cu, None, {})
354 354
355 355 try:
356 356 log.debug("Trying to open URL %s", cleaned_uri)
357 357 resp = o.open(req)
358 358 if resp.code != 200:
359 359 raise exceptions.URLError('Return Code is not 200')
360 360 except Exception as e:
361 361 log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
362 362 # means it cannot be cloned
363 363 raise exceptions.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
364 364
365 365 # now check if it's a proper hg repo, but don't do it for svn
366 366 try:
367 367 if _proto == 'svn':
368 368 pass
369 369 else:
370 370 # check for pure hg repos
371 371 log.debug(
372 372 "Verifying if URL is a Mercurial repository: %s",
373 373 cleaned_uri)
374 374 httppeer(make_ui_from_config(config), url).lookup('tip')
375 375 except Exception as e:
376 376 log.warning("URL is not a valid Mercurial repository: %s",
377 377 cleaned_uri)
378 378 raise exceptions.URLError(
379 379 "url [%s] does not look like an hg repo org_exc: %s"
380 380 % (cleaned_uri, e))
381 381
382 382 log.info("URL is a valid Mercurial repository: %s", cleaned_uri)
383 383 return True
384 384
385 385 @reraise_safe_exceptions
386 386 def diff(
387 387 self, wire, rev1, rev2, file_filter, opt_git, opt_ignorews,
388 388 context):
389 389 repo = self._factory.repo(wire)
390 390
391 391 if file_filter:
392 filter = match(file_filter[0], '', [file_filter[1]])
392 match_filter = match(file_filter[0], '', [file_filter[1]])
393 393 else:
394 filter = file_filter
394 match_filter = file_filter
395 395 opts = diffopts(git=opt_git, ignorews=opt_ignorews, context=context)
396 396
397 397 try:
398 398 return "".join(patch.diff(
399 repo, node1=rev1, node2=rev2, match=filter, opts=opts))
399 repo, node1=rev1, node2=rev2, match=match_filter, opts=opts))
400 400 except RepoLookupError:
401 401 raise exceptions.LookupException()
402 402
403 403 @reraise_safe_exceptions
404 404 def file_history(self, wire, revision, path, limit):
405 405 repo = self._factory.repo(wire)
406 406
407 407 ctx = repo[revision]
408 408 fctx = ctx.filectx(path)
409 409
410 410 def history_iter():
411 411 limit_rev = fctx.rev()
412 412 for obj in reversed(list(fctx.filelog())):
413 413 obj = fctx.filectx(obj)
414 414 if limit_rev >= obj.rev():
415 415 yield obj
416 416
417 417 history = []
418 418 for cnt, obj in enumerate(history_iter()):
419 419 if limit and cnt >= limit:
420 420 break
421 421 history.append(hex(obj.node()))
422 422
423 423 return [x for x in history]
424 424
425 425 @reraise_safe_exceptions
426 426 def file_history_untill(self, wire, revision, path, limit):
427 427 repo = self._factory.repo(wire)
428 428 ctx = repo[revision]
429 429 fctx = ctx.filectx(path)
430 430
431 431 file_log = list(fctx.filelog())
432 432 if limit:
433 433 # Limit to the last n items
434 434 file_log = file_log[-limit:]
435 435
436 436 return [hex(fctx.filectx(cs).node()) for cs in reversed(file_log)]
437 437
438 438 @reraise_safe_exceptions
439 439 def fctx_annotate(self, wire, revision, path):
440 440 repo = self._factory.repo(wire)
441 441 ctx = repo[revision]
442 442 fctx = ctx.filectx(path)
443 443
444 444 result = []
445 445 for i, annotate_data in enumerate(fctx.annotate()):
446 446 ln_no = i + 1
447 447 node_info, content = annotate_data
448 448 sha = hex(node_info[0].node())
449 449 result.append((ln_no, sha, content))
450 450 return result
451 451
452 452 @reraise_safe_exceptions
453 453 def fctx_data(self, wire, revision, path):
454 454 repo = self._factory.repo(wire)
455 455 ctx = repo[revision]
456 456 fctx = ctx.filectx(path)
457 457 return fctx.data()
458 458
459 459 @reraise_safe_exceptions
460 460 def fctx_flags(self, wire, revision, path):
461 461 repo = self._factory.repo(wire)
462 462 ctx = repo[revision]
463 463 fctx = ctx.filectx(path)
464 464 return fctx.flags()
465 465
466 466 @reraise_safe_exceptions
467 467 def fctx_size(self, wire, revision, path):
468 468 repo = self._factory.repo(wire)
469 469 ctx = repo[revision]
470 470 fctx = ctx.filectx(path)
471 471 return fctx.size()
472 472
473 473 @reraise_safe_exceptions
474 474 def get_all_commit_ids(self, wire, name):
475 475 repo = self._factory.repo(wire)
476 476 revs = repo.filtered(name).changelog.index
477 477 return map(lambda x: hex(x[7]), revs)[:-1]
478 478
479 479 @reraise_safe_exceptions
480 480 def get_config_value(self, wire, section, name, untrusted=False):
481 481 repo = self._factory.repo(wire)
482 482 return repo.ui.config(section, name, untrusted=untrusted)
483 483
484 484 @reraise_safe_exceptions
485 485 def get_config_bool(self, wire, section, name, untrusted=False):
486 486 repo = self._factory.repo(wire)
487 487 return repo.ui.configbool(section, name, untrusted=untrusted)
488 488
489 489 @reraise_safe_exceptions
490 490 def get_config_list(self, wire, section, name, untrusted=False):
491 491 repo = self._factory.repo(wire)
492 492 return repo.ui.configlist(section, name, untrusted=untrusted)
493 493
494 494 @reraise_safe_exceptions
495 495 def is_large_file(self, wire, path):
496 496 return largefiles.lfutil.isstandin(path)
497 497
498 498 @reraise_safe_exceptions
499 499 def in_store(self, wire, sha):
500 500 repo = self._factory.repo(wire)
501 501 return largefiles.lfutil.instore(repo, sha)
502 502
503 503 @reraise_safe_exceptions
504 504 def in_user_cache(self, wire, sha):
505 505 repo = self._factory.repo(wire)
506 506 return largefiles.lfutil.inusercache(repo.ui, sha)
507 507
508 508 @reraise_safe_exceptions
509 509 def store_path(self, wire, sha):
510 510 repo = self._factory.repo(wire)
511 511 return largefiles.lfutil.storepath(repo, sha)
512 512
513 513 @reraise_safe_exceptions
514 514 def link(self, wire, sha, path):
515 515 repo = self._factory.repo(wire)
516 516 largefiles.lfutil.link(
517 517 largefiles.lfutil.usercachepath(repo.ui, sha), path)
518 518
519 519 @reraise_safe_exceptions
520 520 def localrepository(self, wire, create=False):
521 521 self._factory.repo(wire, create=create)
522 522
523 523 @reraise_safe_exceptions
524 524 def lookup(self, wire, revision, both):
525 525 # TODO Paris: Ugly hack to "deserialize" long for msgpack
526 526 if isinstance(revision, float):
527 527 revision = long(revision)
528 528 repo = self._factory.repo(wire)
529 529 try:
530 530 ctx = repo[revision]
531 531 except RepoLookupError:
532 532 raise exceptions.LookupException(revision)
533 533 except LookupError as e:
534 534 raise exceptions.LookupException(e.name)
535 535
536 536 if not both:
537 537 return ctx.hex()
538 538
539 539 ctx = repo[ctx.hex()]
540 540 return ctx.hex(), ctx.rev()
541 541
542 542 @reraise_safe_exceptions
543 543 def pull(self, wire, url, commit_ids=None):
544 544 repo = self._factory.repo(wire)
545 545 remote = peer(repo, {}, url)
546 546 if commit_ids:
547 547 commit_ids = [bin(commit_id) for commit_id in commit_ids]
548 548
549 549 return exchange.pull(
550 550 repo, remote, heads=commit_ids, force=None).cgresult
551 551
552 552 @reraise_safe_exceptions
553 553 def revision(self, wire, rev):
554 554 repo = self._factory.repo(wire)
555 555 ctx = repo[rev]
556 556 return ctx.rev()
557 557
558 558 @reraise_safe_exceptions
559 559 def rev_range(self, wire, filter):
560 560 repo = self._factory.repo(wire)
561 561 revisions = [rev for rev in revrange(repo, filter)]
562 562 return revisions
563 563
564 564 @reraise_safe_exceptions
565 565 def rev_range_hash(self, wire, node):
566 566 repo = self._factory.repo(wire)
567 567
568 568 def get_revs(repo, rev_opt):
569 569 if rev_opt:
570 570 revs = revrange(repo, rev_opt)
571 571 if len(revs) == 0:
572 572 return (nullrev, nullrev)
573 573 return max(revs), min(revs)
574 574 else:
575 575 return len(repo) - 1, 0
576 576
577 577 stop, start = get_revs(repo, [node + ':'])
578 578 revs = [hex(repo[r].node()) for r in xrange(start, stop + 1)]
579 579 return revs
580 580
581 581 @reraise_safe_exceptions
582 582 def revs_from_revspec(self, wire, rev_spec, *args, **kwargs):
583 583 other_path = kwargs.pop('other_path', None)
584 584
585 585 # case when we want to compare two independent repositories
586 586 if other_path and other_path != wire["path"]:
587 587 baseui = self._factory._create_config(wire["config"])
588 588 repo = unionrepo.unionrepository(baseui, other_path, wire["path"])
589 589 else:
590 590 repo = self._factory.repo(wire)
591 591 return list(repo.revs(rev_spec, *args))
592 592
593 593 @reraise_safe_exceptions
594 594 def strip(self, wire, revision, update, backup):
595 595 repo = self._factory.repo(wire)
596 596 ctx = repo[revision]
597 597 hgext_strip(
598 598 repo.baseui, repo, ctx.node(), update=update, backup=backup)
599 599
600 600 @reraise_safe_exceptions
601 601 def tag(self, wire, name, revision, message, local, user,
602 602 tag_time, tag_timezone):
603 603 repo = self._factory.repo(wire)
604 604 ctx = repo[revision]
605 605 node = ctx.node()
606 606
607 607 date = (tag_time, tag_timezone)
608 608 try:
609 609 repo.tag(name, node, message, local, user, date)
610 610 except Abort as e:
611 611 log.exception("Tag operation aborted")
612 612 # Exception can contain unicode which we convert
613 613 raise exceptions.AbortException(repr(e))
614 614
615 615 @reraise_safe_exceptions
616 616 def tags(self, wire):
617 617 repo = self._factory.repo(wire)
618 618 return repo.tags()
619 619
620 620 @reraise_safe_exceptions
621 621 def update(self, wire, node=None, clean=False):
622 622 repo = self._factory.repo(wire)
623 623 baseui = self._factory._create_config(wire['config'])
624 624 commands.update(baseui, repo, node=node, clean=clean)
625 625
626 626 @reraise_safe_exceptions
627 627 def identify(self, wire):
628 628 repo = self._factory.repo(wire)
629 629 baseui = self._factory._create_config(wire['config'])
630 630 output = io.BytesIO()
631 631 baseui.write = output.write
632 632 # This is required to get a full node id
633 633 baseui.debugflag = True
634 634 commands.identify(baseui, repo, id=True)
635 635
636 636 return output.getvalue()
637 637
638 638 @reraise_safe_exceptions
639 639 def pull_cmd(self, wire, source, bookmark=None, branch=None, revision=None,
640 640 hooks=True):
641 641 repo = self._factory.repo(wire)
642 642 baseui = self._factory._create_config(wire['config'], hooks=hooks)
643 643
644 644 # Mercurial internally has a lot of logic that checks ONLY if
645 645 # option is defined, we just pass those if they are defined then
646 646 opts = {}
647 647 if bookmark:
648 648 opts['bookmark'] = bookmark
649 649 if branch:
650 650 opts['branch'] = branch
651 651 if revision:
652 652 opts['rev'] = revision
653 653
654 654 commands.pull(baseui, repo, source, **opts)
655 655
656 656 @reraise_safe_exceptions
657 657 def heads(self, wire, branch=None):
658 658 repo = self._factory.repo(wire)
659 659 baseui = self._factory._create_config(wire['config'])
660 660 output = io.BytesIO()
661 661
662 662 def write(data, **unused_kwargs):
663 663 output.write(data)
664 664
665 665 baseui.write = write
666 666 if branch:
667 667 args = [branch]
668 668 else:
669 669 args = []
670 670 commands.heads(baseui, repo, template='{node} ', *args)
671 671
672 672 return output.getvalue()
673 673
674 674 @reraise_safe_exceptions
675 675 def ancestor(self, wire, revision1, revision2):
676 676 repo = self._factory.repo(wire)
677 677 baseui = self._factory._create_config(wire['config'])
678 678 output = io.BytesIO()
679 679 baseui.write = output.write
680 680 commands.debugancestor(baseui, repo, revision1, revision2)
681 681
682 682 return output.getvalue()
683 683
684 684 @reraise_safe_exceptions
685 685 def push(self, wire, revisions, dest_path, hooks=True,
686 686 push_branches=False):
687 687 repo = self._factory.repo(wire)
688 688 baseui = self._factory._create_config(wire['config'], hooks=hooks)
689 689 commands.push(baseui, repo, dest=dest_path, rev=revisions,
690 690 new_branch=push_branches)
691 691
692 692 @reraise_safe_exceptions
693 693 def merge(self, wire, revision):
694 694 repo = self._factory.repo(wire)
695 695 baseui = self._factory._create_config(wire['config'])
696 696 repo.ui.setconfig('ui', 'merge', 'internal:dump')
697 697
698 698 # In case of sub repositories are used mercurial prompts the user in
699 699 # case of merge conflicts or different sub repository sources. By
700 700 # setting the interactive flag to `False` mercurial doesn't prompt the
701 701 # used but instead uses a default value.
702 702 repo.ui.setconfig('ui', 'interactive', False)
703 703
704 704 commands.merge(baseui, repo, rev=revision)
705 705
706 706 @reraise_safe_exceptions
707 707 def commit(self, wire, message, username):
708 708 repo = self._factory.repo(wire)
709 709 baseui = self._factory._create_config(wire['config'])
710 710 repo.ui.setconfig('ui', 'username', username)
711 711 commands.commit(baseui, repo, message=message)
712 712
713 713 @reraise_safe_exceptions
714 714 def rebase(self, wire, source=None, dest=None, abort=False):
715 715 repo = self._factory.repo(wire)
716 716 baseui = self._factory._create_config(wire['config'])
717 717 repo.ui.setconfig('ui', 'merge', 'internal:dump')
718 718 rebase.rebase(
719 719 baseui, repo, base=source, dest=dest, abort=abort, keep=not abort)
720 720
721 721 @reraise_safe_exceptions
722 722 def bookmark(self, wire, bookmark, revision=None):
723 723 repo = self._factory.repo(wire)
724 724 baseui = self._factory._create_config(wire['config'])
725 725 commands.bookmark(baseui, repo, bookmark, rev=revision, force=True)
@@ -1,62 +1,62 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 """
19 19 Mercurial libs compatibility
20 20 """
21 21
22 22 import mercurial
23 23 import mercurial.demandimport
24 24 # patch demandimport, due to bug in mercurial when it always triggers
25 25 # demandimport.enable()
26 26 mercurial.demandimport.enable = lambda *args, **kwargs: 1
27 27
28 28 from mercurial import ui
29 29 from mercurial import patch
30 30 from mercurial import config
31 31 from mercurial import extensions
32 32 from mercurial import scmutil
33 33 from mercurial import archival
34 34 from mercurial import discovery
35 35 from mercurial import unionrepo
36 36 from mercurial import localrepo
37 37 from mercurial import merge as hg_merge
38 38 from mercurial import subrepo
39 39
40 40 from mercurial.commands import clone, nullid, pull
41 41 from mercurial.context import memctx, memfilectx
42 42 from mercurial.error import (
43 43 LookupError, RepoError, RepoLookupError, Abort, InterventionRequired,
44 44 RequirementError)
45 45 from mercurial.hgweb import hgweb_mod
46 46 from mercurial.localrepo import localrepository
47 47 from mercurial.match import match
48 48 from mercurial.mdiff import diffopts
49 49 from mercurial.node import bin, hex
50 50 from mercurial.encoding import tolocal
51 51 from mercurial.discovery import findcommonoutgoing
52 52 from mercurial.hg import peer
53 53 from mercurial.httppeer import httppeer
54 54 from mercurial.util import url as hg_url
55 55 from mercurial.scmutil import revrange
56 56 from mercurial.node import nullrev
57 57 from mercurial import exchange
58 58 from hgext import largefiles
59 59
60 60 # those authnadlers are patched for python 2.6.5 bug an
61 61 # infinit looping when given invalid resources
62 62 from mercurial.url import httpbasicauthhandler, httpdigestauthhandler
@@ -1,134 +1,134 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 """
19 19 Adjustments to Mercurial
20 20
21 21 Intentionally kept separate from `hgcompat` and `hg`, so that these patches can
22 22 be applied without having to import the whole Mercurial machinery.
23 23
24 24 Imports are function local, so that just importing this module does not cause
25 25 side-effects other than these functions being defined.
26 26 """
27 27
28 28 import logging
29 29
30 30
31 31 def patch_largefiles_capabilities():
32 32 """
33 33 Patches the capabilities function in the largefiles extension.
34 34 """
35 35 from vcsserver import hgcompat
36 36 lfproto = hgcompat.largefiles.proto
37 37 wrapper = _dynamic_capabilities_wrapper(
38 38 lfproto, hgcompat.extensions.extensions)
39 39 lfproto.capabilities = wrapper
40 40
41 41
42 42 def _dynamic_capabilities_wrapper(lfproto, extensions):
43 43
44 44 wrapped_capabilities = lfproto.capabilities
45 45 logger = logging.getLogger('vcsserver.hg')
46 46
47 47 def _dynamic_capabilities(repo, proto):
48 48 """
49 49 Adds dynamic behavior, so that the capability is only added if the
50 50 extension is enabled in the current ui object.
51 51 """
52 52 if 'largefiles' in dict(extensions(repo.ui)):
53 53 logger.debug('Extension largefiles enabled')
54 54 calc_capabilities = wrapped_capabilities
55 55 else:
56 56 logger.debug('Extension largefiles disabled')
57 57 calc_capabilities = lfproto.capabilitiesorig
58 58 return calc_capabilities(repo, proto)
59 59
60 60 return _dynamic_capabilities
61 61
62 62
63 63 def patch_subrepo_type_mapping():
64 64 from collections import defaultdict
65 65 from hgcompat import subrepo
66 66 from exceptions import SubrepoMergeException
67 67
68 68 class NoOpSubrepo(subrepo.abstractsubrepo):
69 69
70 70 def __init__(self, ctx, path, *args, **kwargs):
71 71 """Initialize abstractsubrepo part
72 72
73 73 ``ctx`` is the context referring this subrepository in the
74 74 parent repository.
75 75
76 76 ``path`` is the path to this subrepository as seen from
77 77 innermost repository.
78 78 """
79 79 self.ui = ctx.repo().ui
80 80 self._ctx = ctx
81 81 self._path = path
82 82
83 83 def storeclean(self, path):
84 84 """
85 85 returns true if the repository has not changed since it was last
86 86 cloned from or pushed to a given repository.
87 87 """
88 88 return True
89 89
90 90 def dirty(self, ignoreupdate=False):
91 91 """returns true if the dirstate of the subrepo is dirty or does not
92 92 match current stored state. If ignoreupdate is true, only check
93 93 whether the subrepo has uncommitted changes in its dirstate.
94 94 """
95 95 return False
96 96
97 97 def basestate(self):
98 98 """current working directory base state, disregarding .hgsubstate
99 99 state and working directory modifications"""
100 100 substate = subrepo.state(self._ctx, self.ui)
101 101 file_system_path, rev, repotype = substate.get(self._path)
102 102 return rev
103 103
104 104 def remove(self):
105 105 """remove the subrepo
106 106
107 107 (should verify the dirstate is not dirty first)
108 108 """
109 109 pass
110 110
111 111 def get(self, state, overwrite=False):
112 112 """run whatever commands are needed to put the subrepo into
113 113 this state
114 114 """
115 115 pass
116 116
117 117 def merge(self, state):
118 118 """merge currently-saved state with the new state."""
119 119 raise SubrepoMergeException()
120 120
121 121 def push(self, opts):
122 122 """perform whatever action is analogous to 'hg push'
123 123
124 124 This may be a no-op on some systems.
125 125 """
126 126 pass
127 127
128 128 # Patch subrepo type mapping to always return our NoOpSubrepo class
129 129 # whenever a subrepo class is looked up.
130 130 subrepo.types = {
131 131 'hg': NoOpSubrepo,
132 132 'git': NoOpSubrepo,
133 133 'svn': NoOpSubrepo
134 134 }
@@ -1,372 +1,372 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # RhodeCode VCSServer provides access to different vcs backends via network.
4 # Copyright (C) 2014-2016 RodeCode GmbH
4 # Copyright (C) 2014-2017 RodeCode GmbH
5 5 #
6 6 # This program is free software; you can redistribute it and/or modify
7 7 # it under the terms of the GNU General Public License as published by
8 8 # the Free Software Foundation; either version 3 of the License, or
9 9 # (at your option) any later version.
10 10 #
11 11 # This program is distributed in the hope that it will be useful,
12 12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 14 # GNU General Public License for more details.
15 15 #
16 16 # You should have received a copy of the GNU General Public License
17 17 # along with this program; if not, write to the Free Software Foundation,
18 18 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 19
20 20 import collections
21 21 import importlib
22 22 import io
23 23 import json
24 24 import subprocess
25 25 import sys
26 26 from httplib import HTTPConnection
27 27
28 28
29 29 import mercurial.scmutil
30 30 import mercurial.node
31 31 import Pyro4
32 32 import simplejson as json
33 33
34 34 from vcsserver import exceptions
35 35
36 36
37 37 class HooksHttpClient(object):
38 38 connection = None
39 39
40 40 def __init__(self, hooks_uri):
41 41 self.hooks_uri = hooks_uri
42 42
43 43 def __call__(self, method, extras):
44 44 connection = HTTPConnection(self.hooks_uri)
45 45 body = self._serialize(method, extras)
46 46 connection.request('POST', '/', body)
47 47 response = connection.getresponse()
48 48 return json.loads(response.read())
49 49
50 50 def _serialize(self, hook_name, extras):
51 51 data = {
52 52 'method': hook_name,
53 53 'extras': extras
54 54 }
55 55 return json.dumps(data)
56 56
57 57
58 58 class HooksDummyClient(object):
59 59 def __init__(self, hooks_module):
60 60 self._hooks_module = importlib.import_module(hooks_module)
61 61
62 62 def __call__(self, hook_name, extras):
63 63 with self._hooks_module.Hooks() as hooks:
64 64 return getattr(hooks, hook_name)(extras)
65 65
66 66
67 67 class HooksPyro4Client(object):
68 68 def __init__(self, hooks_uri):
69 69 self.hooks_uri = hooks_uri
70 70
71 71 def __call__(self, hook_name, extras):
72 72 with Pyro4.Proxy(self.hooks_uri) as hooks:
73 73 return getattr(hooks, hook_name)(extras)
74 74
75 75
76 76 class RemoteMessageWriter(object):
77 77 """Writer base class."""
78 78 def write(message):
79 79 raise NotImplementedError()
80 80
81 81
82 82 class HgMessageWriter(RemoteMessageWriter):
83 83 """Writer that knows how to send messages to mercurial clients."""
84 84
85 85 def __init__(self, ui):
86 86 self.ui = ui
87 87
88 88 def write(self, message):
89 89 # TODO: Check why the quiet flag is set by default.
90 90 old = self.ui.quiet
91 91 self.ui.quiet = False
92 92 self.ui.status(message.encode('utf-8'))
93 93 self.ui.quiet = old
94 94
95 95
96 96 class GitMessageWriter(RemoteMessageWriter):
97 97 """Writer that knows how to send messages to git clients."""
98 98
99 99 def __init__(self, stdout=None):
100 100 self.stdout = stdout or sys.stdout
101 101
102 102 def write(self, message):
103 103 self.stdout.write(message.encode('utf-8'))
104 104
105 105
106 106 def _handle_exception(result):
107 107 exception_class = result.get('exception')
108 108 if exception_class == 'HTTPLockedRC':
109 109 raise exceptions.RepositoryLockedException(*result['exception_args'])
110 110 elif exception_class == 'RepositoryError':
111 111 raise exceptions.VcsException(*result['exception_args'])
112 112 elif exception_class:
113 113 raise Exception('Got remote exception "%s" with args "%s"' %
114 114 (exception_class, result['exception_args']))
115 115
116 116
117 117 def _get_hooks_client(extras):
118 118 if 'hooks_uri' in extras:
119 119 protocol = extras.get('hooks_protocol')
120 120 return (
121 121 HooksHttpClient(extras['hooks_uri'])
122 122 if protocol == 'http'
123 123 else HooksPyro4Client(extras['hooks_uri'])
124 124 )
125 125 else:
126 126 return HooksDummyClient(extras['hooks_module'])
127 127
128 128
129 129 def _call_hook(hook_name, extras, writer):
130 130 hooks = _get_hooks_client(extras)
131 131 result = hooks(hook_name, extras)
132 132 writer.write(result['output'])
133 133 _handle_exception(result)
134 134
135 135 return result['status']
136 136
137 137
138 138 def _extras_from_ui(ui):
139 139 extras = json.loads(ui.config('rhodecode', 'RC_SCM_DATA'))
140 140 return extras
141 141
142 142
143 143 def repo_size(ui, repo, **kwargs):
144 144 return _call_hook('repo_size', _extras_from_ui(ui), HgMessageWriter(ui))
145 145
146 146
147 147 def pre_pull(ui, repo, **kwargs):
148 148 return _call_hook('pre_pull', _extras_from_ui(ui), HgMessageWriter(ui))
149 149
150 150
151 151 def post_pull(ui, repo, **kwargs):
152 152 return _call_hook('post_pull', _extras_from_ui(ui), HgMessageWriter(ui))
153 153
154 154
155 155 def pre_push(ui, repo, **kwargs):
156 156 return _call_hook('pre_push', _extras_from_ui(ui), HgMessageWriter(ui))
157 157
158 158
159 159 # N.B.(skreft): the two functions below were taken and adapted from
160 160 # rhodecode.lib.vcs.remote.handle_git_pre_receive
161 161 # They are required to compute the commit_ids
162 162 def _get_revs(repo, rev_opt):
163 163 revs = [rev for rev in mercurial.scmutil.revrange(repo, rev_opt)]
164 164 if len(revs) == 0:
165 165 return (mercurial.node.nullrev, mercurial.node.nullrev)
166 166
167 167 return max(revs), min(revs)
168 168
169 169
170 170 def _rev_range_hash(repo, node):
171 171 stop, start = _get_revs(repo, [node + ':'])
172 172 revs = [mercurial.node.hex(repo[r].node()) for r in xrange(start, stop + 1)]
173 173
174 174 return revs
175 175
176 176
177 177 def post_push(ui, repo, node, **kwargs):
178 178 commit_ids = _rev_range_hash(repo, node)
179 179
180 180 extras = _extras_from_ui(ui)
181 181 extras['commit_ids'] = commit_ids
182 182
183 183 return _call_hook('post_push', extras, HgMessageWriter(ui))
184 184
185 185
186 186 # backward compat
187 187 log_pull_action = post_pull
188 188
189 189 # backward compat
190 190 log_push_action = post_push
191 191
192 192
193 193 def handle_git_pre_receive(unused_repo_path, unused_revs, unused_env):
194 194 """
195 195 Old hook name: keep here for backward compatibility.
196 196
197 197 This is only required when the installed git hooks are not upgraded.
198 198 """
199 199 pass
200 200
201 201
202 202 def handle_git_post_receive(unused_repo_path, unused_revs, unused_env):
203 203 """
204 204 Old hook name: keep here for backward compatibility.
205 205
206 206 This is only required when the installed git hooks are not upgraded.
207 207 """
208 208 pass
209 209
210 210
211 211 HookResponse = collections.namedtuple('HookResponse', ('status', 'output'))
212 212
213 213
214 214 def git_pre_pull(extras):
215 215 """
216 216 Pre pull hook.
217 217
218 218 :param extras: dictionary containing the keys defined in simplevcs
219 219 :type extras: dict
220 220
221 221 :return: status code of the hook. 0 for success.
222 222 :rtype: int
223 223 """
224 224 if 'pull' not in extras['hooks']:
225 225 return HookResponse(0, '')
226 226
227 227 stdout = io.BytesIO()
228 228 try:
229 229 status = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
230 230 except Exception as error:
231 231 status = 128
232 232 stdout.write('ERROR: %s\n' % str(error))
233 233
234 234 return HookResponse(status, stdout.getvalue())
235 235
236 236
237 237 def git_post_pull(extras):
238 238 """
239 239 Post pull hook.
240 240
241 241 :param extras: dictionary containing the keys defined in simplevcs
242 242 :type extras: dict
243 243
244 244 :return: status code of the hook. 0 for success.
245 245 :rtype: int
246 246 """
247 247 if 'pull' not in extras['hooks']:
248 248 return HookResponse(0, '')
249 249
250 250 stdout = io.BytesIO()
251 251 try:
252 252 status = _call_hook('post_pull', extras, GitMessageWriter(stdout))
253 253 except Exception as error:
254 254 status = 128
255 255 stdout.write('ERROR: %s\n' % error)
256 256
257 257 return HookResponse(status, stdout.getvalue())
258 258
259 259
260 260 def git_pre_receive(unused_repo_path, unused_revs, env):
261 261 """
262 262 Pre push hook.
263 263
264 264 :param extras: dictionary containing the keys defined in simplevcs
265 265 :type extras: dict
266 266
267 267 :return: status code of the hook. 0 for success.
268 268 :rtype: int
269 269 """
270 270 extras = json.loads(env['RC_SCM_DATA'])
271 271 if 'push' not in extras['hooks']:
272 272 return 0
273 273 return _call_hook('pre_push', extras, GitMessageWriter())
274 274
275 275
276 276 def _run_command(arguments):
277 277 """
278 278 Run the specified command and return the stdout.
279 279
280 280 :param arguments: sequence of program arugments (including the program name)
281 281 :type arguments: list[str]
282 282 """
283 283 # TODO(skreft): refactor this method and all the other similar ones.
284 284 # Probably this should be using subprocessio.
285 285 process = subprocess.Popen(
286 286 arguments, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
287 287 stdout, _ = process.communicate()
288 288
289 289 if process.returncode != 0:
290 290 raise Exception(
291 291 'Command %s exited with exit code %s' % (arguments,
292 292 process.returncode))
293 293
294 294 return stdout
295 295
296 296
297 297 def git_post_receive(unused_repo_path, revision_lines, env):
298 298 """
299 299 Post push hook.
300 300
301 301 :param extras: dictionary containing the keys defined in simplevcs
302 302 :type extras: dict
303 303
304 304 :return: status code of the hook. 0 for success.
305 305 :rtype: int
306 306 """
307 307 extras = json.loads(env['RC_SCM_DATA'])
308 308 if 'push' not in extras['hooks']:
309 309 return 0
310 310
311 311 rev_data = []
312 312 for revision_line in revision_lines:
313 313 old_rev, new_rev, ref = revision_line.strip().split(' ')
314 314 ref_data = ref.split('/', 2)
315 315 if ref_data[1] in ('tags', 'heads'):
316 316 rev_data.append({
317 317 'old_rev': old_rev,
318 318 'new_rev': new_rev,
319 319 'ref': ref,
320 320 'type': ref_data[1],
321 321 'name': ref_data[2],
322 322 })
323 323
324 324 git_revs = []
325 325
326 326 # N.B.(skreft): it is ok to just call git, as git before calling a
327 327 # subcommand sets the PATH environment variable so that it point to the
328 328 # correct version of the git executable.
329 329 empty_commit_id = '0' * 40
330 330 for push_ref in rev_data:
331 331 type_ = push_ref['type']
332 332 if type_ == 'heads':
333 333 if push_ref['old_rev'] == empty_commit_id:
334 334
335 335 # Fix up head revision if needed
336 336 cmd = ['git', 'show', 'HEAD']
337 337 try:
338 338 _run_command(cmd)
339 339 except Exception:
340 340 cmd = ['git', 'symbolic-ref', 'HEAD',
341 341 'refs/heads/%s' % push_ref['name']]
342 342 print "Setting default branch to %s" % push_ref['name']
343 343 _run_command(cmd)
344 344
345 345 cmd = ['git', 'for-each-ref', '--format=%(refname)',
346 346 'refs/heads/*']
347 347 heads = _run_command(cmd)
348 348 heads = heads.replace(push_ref['ref'], '')
349 349 heads = ' '.join(head for head in heads.splitlines() if head)
350 350 cmd = ['git', 'log', '--reverse', '--pretty=format:%H',
351 351 '--', push_ref['new_rev'], '--not', heads]
352 352 git_revs.extend(_run_command(cmd).splitlines())
353 353 elif push_ref['new_rev'] == empty_commit_id:
354 354 # delete branch case
355 355 git_revs.append('delete_branch=>%s' % push_ref['name'])
356 356 else:
357 357 cmd = ['git', 'log',
358 358 '{old_rev}..{new_rev}'.format(**push_ref),
359 359 '--reverse', '--pretty=format:%H']
360 360 git_revs.extend(_run_command(cmd).splitlines())
361 361 elif type_ == 'tags':
362 362 git_revs.append('tag=>%s' % push_ref['name'])
363 363
364 364 extras['commit_ids'] = git_revs
365 365
366 366 if 'repo_size' in extras['hooks']:
367 367 try:
368 368 _call_hook('repo_size', extras, GitMessageWriter())
369 369 except:
370 370 pass
371 371
372 372 return _call_hook('post_push', extras, GitMessageWriter())
@@ -1,376 +1,408 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import base64
19 19 import locale
20 20 import logging
21 21 import uuid
22 22 import wsgiref.util
23 import traceback
23 24 from itertools import chain
24 25
25 26 import msgpack
26 27 from beaker.cache import CacheManager
27 28 from beaker.util import parse_cache_config_options
28 29 from pyramid.config import Configurator
29 30 from pyramid.wsgi import wsgiapp
30 31
31 32 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
32 33 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
33 34 from vcsserver.echo_stub.echo_app import EchoApp
34 35 from vcsserver.exceptions import HTTPRepoLocked
35 36 from vcsserver.server import VcsServer
36 37
37 38 try:
38 39 from vcsserver.git import GitFactory, GitRemote
39 40 except ImportError:
40 41 GitFactory = None
41 42 GitRemote = None
42 43 try:
43 44 from vcsserver.hg import MercurialFactory, HgRemote
44 45 except ImportError:
45 46 MercurialFactory = None
46 47 HgRemote = None
47 48 try:
48 49 from vcsserver.svn import SubversionFactory, SvnRemote
49 50 except ImportError:
50 51 SubversionFactory = None
51 52 SvnRemote = None
52 53
53 54 log = logging.getLogger(__name__)
54 55
55 56
56 57 class VCS(object):
57 58 def __init__(self, locale=None, cache_config=None):
58 59 self.locale = locale
59 60 self.cache_config = cache_config
60 61 self._configure_locale()
61 62 self._initialize_cache()
62 63
63 64 if GitFactory and GitRemote:
64 65 git_repo_cache = self.cache.get_cache_region(
65 66 'git', region='repo_object')
66 67 git_factory = GitFactory(git_repo_cache)
67 68 self._git_remote = GitRemote(git_factory)
68 69 else:
69 70 log.info("Git client import failed")
70 71
71 72 if MercurialFactory and HgRemote:
72 73 hg_repo_cache = self.cache.get_cache_region(
73 74 'hg', region='repo_object')
74 75 hg_factory = MercurialFactory(hg_repo_cache)
75 76 self._hg_remote = HgRemote(hg_factory)
76 77 else:
77 78 log.info("Mercurial client import failed")
78 79
79 80 if SubversionFactory and SvnRemote:
80 81 svn_repo_cache = self.cache.get_cache_region(
81 82 'svn', region='repo_object')
82 83 svn_factory = SubversionFactory(svn_repo_cache)
83 84 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
84 85 else:
85 86 log.info("Subversion client import failed")
86 87
87 88 self._vcsserver = VcsServer()
88 89
89 90 def _initialize_cache(self):
90 91 cache_config = parse_cache_config_options(self.cache_config)
91 92 log.info('Initializing beaker cache: %s' % cache_config)
92 93 self.cache = CacheManager(**cache_config)
93 94
94 95 def _configure_locale(self):
95 96 if self.locale:
96 97 log.info('Settings locale: `LC_ALL` to %s' % self.locale)
97 98 else:
98 99 log.info(
99 100 'Configuring locale subsystem based on environment variables')
100 101 try:
101 102 # If self.locale is the empty string, then the locale
102 103 # module will use the environment variables. See the
103 104 # documentation of the package `locale`.
104 105 locale.setlocale(locale.LC_ALL, self.locale)
105 106
106 107 language_code, encoding = locale.getlocale()
107 108 log.info(
108 109 'Locale set to language code "%s" with encoding "%s".',
109 110 language_code, encoding)
110 111 except locale.Error:
111 112 log.exception(
112 113 'Cannot set locale, not configuring the locale system')
113 114
114 115
115 116 class WsgiProxy(object):
116 117 def __init__(self, wsgi):
117 118 self.wsgi = wsgi
118 119
119 120 def __call__(self, environ, start_response):
120 121 input_data = environ['wsgi.input'].read()
121 122 input_data = msgpack.unpackb(input_data)
122 123
123 124 error = None
124 125 try:
125 126 data, status, headers = self.wsgi.handle(
126 127 input_data['environment'], input_data['input_data'],
127 128 *input_data['args'], **input_data['kwargs'])
128 129 except Exception as e:
129 130 data, status, headers = [], None, None
130 131 error = {
131 132 'message': str(e),
132 133 '_vcs_kind': getattr(e, '_vcs_kind', None)
133 134 }
134 135
135 136 start_response(200, {})
136 137 return self._iterator(error, status, headers, data)
137 138
138 139 def _iterator(self, error, status, headers, data):
139 140 initial_data = [
140 141 error,
141 142 status,
142 143 headers,
143 144 ]
144 145
145 146 for d in chain(initial_data, data):
146 147 yield msgpack.packb(d)
147 148
148 149
149 150 class HTTPApplication(object):
150 151 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
151 152
152 153 remote_wsgi = remote_wsgi
153 154 _use_echo_app = False
154 155
155 156 def __init__(self, settings=None):
156 157 self.config = Configurator(settings=settings)
157 locale = settings.get('', 'en_US.UTF-8')
158 locale = settings.get('locale', '') or 'en_US.UTF-8'
158 159 vcs = VCS(locale=locale, cache_config=settings)
159 160 self._remotes = {
160 161 'hg': vcs._hg_remote,
161 162 'git': vcs._git_remote,
162 163 'svn': vcs._svn_remote,
163 164 'server': vcs._vcsserver,
164 165 }
165 166 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
166 167 self._use_echo_app = True
167 168 log.warning("Using EchoApp for VCS operations.")
168 169 self.remote_wsgi = remote_wsgi_stub
169 170 self._configure_settings(settings)
170 171 self._configure()
171 172
172 173 def _configure_settings(self, app_settings):
173 174 """
174 175 Configure the settings module.
175 176 """
176 177 git_path = app_settings.get('git_path', None)
177 178 if git_path:
178 179 settings.GIT_EXECUTABLE = git_path
179 180
180 181 def _configure(self):
181 182 self.config.add_renderer(
182 183 name='msgpack',
183 184 factory=self._msgpack_renderer_factory)
184 185
185 186 self.config.add_route('service', '/_service')
186 187 self.config.add_route('status', '/status')
187 188 self.config.add_route('hg_proxy', '/proxy/hg')
188 189 self.config.add_route('git_proxy', '/proxy/git')
189 190 self.config.add_route('vcs', '/{backend}')
190 191 self.config.add_route('stream_git', '/stream/git/*repo_name')
191 192 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
192 193
193 194 self.config.add_view(
194 195 self.status_view, route_name='status', renderer='json')
195 196 self.config.add_view(
196 197 self.service_view, route_name='service', renderer='msgpack')
197 198
198 199 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
199 200 self.config.add_view(self.git_proxy(), route_name='git_proxy')
200 201 self.config.add_view(
201 self.vcs_view, route_name='vcs', renderer='msgpack')
202 self.vcs_view, route_name='vcs', renderer='msgpack',
203 custom_predicates=[self.is_vcs_view])
202 204
203 205 self.config.add_view(self.hg_stream(), route_name='stream_hg')
204 206 self.config.add_view(self.git_stream(), route_name='stream_git')
207
208 def notfound(request):
209 return {'status': '404 NOT FOUND'}
210 self.config.add_notfound_view(notfound, renderer='json')
211
205 212 self.config.add_view(
206 213 self.handle_vcs_exception, context=Exception,
207 214 custom_predicates=[self.is_vcs_exception])
208 215
216 self.config.add_view(
217 self.general_error_handler, context=Exception)
218
219 self.config.add_tween(
220 'vcsserver.tweens.RequestWrapperTween',
221 )
222
209 223 def wsgi_app(self):
210 224 return self.config.make_wsgi_app()
211 225
212 226 def vcs_view(self, request):
213 227 remote = self._remotes[request.matchdict['backend']]
214 228 payload = msgpack.unpackb(request.body, use_list=True)
215 229 method = payload.get('method')
216 230 params = payload.get('params')
217 231 wire = params.get('wire')
218 232 args = params.get('args')
219 233 kwargs = params.get('kwargs')
220 234 if wire:
221 235 try:
222 236 wire['context'] = uuid.UUID(wire['context'])
223 237 except KeyError:
224 238 pass
225 239 args.insert(0, wire)
226 240
241 log.debug('method called:%s with kwargs:%s', method, kwargs)
227 242 try:
228 243 resp = getattr(remote, method)(*args, **kwargs)
229 244 except Exception as e:
245 tb_info = traceback.format_exc()
246
230 247 type_ = e.__class__.__name__
231 248 if type_ not in self.ALLOWED_EXCEPTIONS:
232 249 type_ = None
233 250
234 251 resp = {
235 252 'id': payload.get('id'),
236 253 'error': {
237 254 'message': e.message,
255 'traceback': tb_info,
238 256 'type': type_
239 257 }
240 258 }
241 259 try:
242 260 resp['error']['_vcs_kind'] = e._vcs_kind
243 261 except AttributeError:
244 262 pass
245 263 else:
246 264 resp = {
247 265 'id': payload.get('id'),
248 266 'result': resp
249 267 }
250 268
251 269 return resp
252 270
253 271 def status_view(self, request):
254 272 return {'status': 'OK'}
255 273
256 274 def service_view(self, request):
257 275 import vcsserver
258 276 payload = msgpack.unpackb(request.body, use_list=True)
259 277 resp = {
260 278 'id': payload.get('id'),
261 279 'result': dict(
262 280 version=vcsserver.__version__,
263 281 config={},
264 282 payload=payload,
265 283 )
266 284 }
267 285 return resp
268 286
269 287 def _msgpack_renderer_factory(self, info):
270 288 def _render(value, system):
271 289 value = msgpack.packb(value)
272 290 request = system.get('request')
273 291 if request is not None:
274 292 response = request.response
275 293 ct = response.content_type
276 294 if ct == response.default_content_type:
277 295 response.content_type = 'application/x-msgpack'
278 296 return value
279 297 return _render
280 298
281 299 def hg_proxy(self):
282 300 @wsgiapp
283 301 def _hg_proxy(environ, start_response):
284 302 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
285 303 return app(environ, start_response)
286 304 return _hg_proxy
287 305
288 306 def git_proxy(self):
289 307 @wsgiapp
290 308 def _git_proxy(environ, start_response):
291 309 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
292 310 return app(environ, start_response)
293 311 return _git_proxy
294 312
295 313 def hg_stream(self):
296 314 if self._use_echo_app:
297 315 @wsgiapp
298 316 def _hg_stream(environ, start_response):
299 317 app = EchoApp('fake_path', 'fake_name', None)
300 318 return app(environ, start_response)
301 319 return _hg_stream
302 320 else:
303 321 @wsgiapp
304 322 def _hg_stream(environ, start_response):
305 323 repo_path = environ['HTTP_X_RC_REPO_PATH']
306 324 repo_name = environ['HTTP_X_RC_REPO_NAME']
307 325 packed_config = base64.b64decode(
308 326 environ['HTTP_X_RC_REPO_CONFIG'])
309 327 config = msgpack.unpackb(packed_config)
310 328 app = scm_app.create_hg_wsgi_app(
311 329 repo_path, repo_name, config)
312 330
313 331 # Consitent path information for hgweb
314 332 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
315 333 environ['REPO_NAME'] = repo_name
316 334 return app(environ, ResponseFilter(start_response))
317 335 return _hg_stream
318 336
319 337 def git_stream(self):
320 338 if self._use_echo_app:
321 339 @wsgiapp
322 340 def _git_stream(environ, start_response):
323 341 app = EchoApp('fake_path', 'fake_name', None)
324 342 return app(environ, start_response)
325 343 return _git_stream
326 344 else:
327 345 @wsgiapp
328 346 def _git_stream(environ, start_response):
329 347 repo_path = environ['HTTP_X_RC_REPO_PATH']
330 348 repo_name = environ['HTTP_X_RC_REPO_NAME']
331 349 packed_config = base64.b64decode(
332 350 environ['HTTP_X_RC_REPO_CONFIG'])
333 351 config = msgpack.unpackb(packed_config)
334 352
335 353 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
336 354 app = scm_app.create_git_wsgi_app(
337 355 repo_path, repo_name, config)
338 356 return app(environ, start_response)
339 357 return _git_stream
340 358
359 def is_vcs_view(self, context, request):
360 """
361 View predicate that returns true if given backend is supported by
362 defined remotes.
363 """
364 backend = request.matchdict.get('backend')
365 return backend in self._remotes
366
341 367 def is_vcs_exception(self, context, request):
342 368 """
343 369 View predicate that returns true if the context object is a VCS
344 370 exception.
345 371 """
346 372 return hasattr(context, '_vcs_kind')
347 373
348 374 def handle_vcs_exception(self, exception, request):
349 375 if exception._vcs_kind == 'repo_locked':
350 376 # Get custom repo-locked status code if present.
351 377 status_code = request.headers.get('X-RC-Locked-Status-Code')
352 378 return HTTPRepoLocked(
353 379 title=exception.message, status_code=status_code)
354 380
355 381 # Re-raise exception if we can not handle it.
356 382 raise exception
357 383
384 def general_error_handler(self, exception, request):
385 log.exception(
386 'error occurred handling this request for path: %s',
387 request.path)
388 raise exception
389
358 390
359 391 class ResponseFilter(object):
360 392
361 393 def __init__(self, start_response):
362 394 self._start_response = start_response
363 395
364 396 def __call__(self, status, response_headers, exc_info=None):
365 397 headers = tuple(
366 398 (h, v) for h, v in response_headers
367 399 if not wsgiref.util.is_hop_by_hop(h))
368 400 return self._start_response(status, headers, exc_info)
369 401
370 402
371 403 def main(global_config, **settings):
372 404 if MercurialFactory:
373 405 hgpatches.patch_largefiles_capabilities()
374 406 hgpatches.patch_subrepo_type_mapping()
375 407 app = HTTPApplication(settings=settings)
376 408 return app.wsgi_app()
@@ -1,508 +1,508 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import atexit
19 19 import locale
20 20 import logging
21 21 import optparse
22 22 import os
23 23 import textwrap
24 24 import threading
25 25 import sys
26 26
27 27 import configobj
28 28 import Pyro4
29 29 from beaker.cache import CacheManager
30 30 from beaker.util import parse_cache_config_options
31 31
32 32 try:
33 33 from vcsserver.git import GitFactory, GitRemote
34 34 except ImportError:
35 35 GitFactory = None
36 36 GitRemote = None
37 37 try:
38 38 from vcsserver.hg import MercurialFactory, HgRemote
39 39 except ImportError:
40 40 MercurialFactory = None
41 41 HgRemote = None
42 42 try:
43 43 from vcsserver.svn import SubversionFactory, SvnRemote
44 44 except ImportError:
45 45 SubversionFactory = None
46 46 SvnRemote = None
47 47
48 48 from server import VcsServer
49 49 from vcsserver import hgpatches, remote_wsgi, settings
50 50 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
51 51
52 52 log = logging.getLogger(__name__)
53 53
54 54 HERE = os.path.dirname(os.path.abspath(__file__))
55 55 SERVER_RUNNING_FILE = None
56 56
57 57
58 58 # HOOKS - inspired by gunicorn #
59 59
60 60 def when_ready(server):
61 61 """
62 62 Called just after the server is started.
63 63 """
64 64
65 65 def _remove_server_running_file():
66 66 if os.path.isfile(SERVER_RUNNING_FILE):
67 67 os.remove(SERVER_RUNNING_FILE)
68 68
69 69 # top up to match to level location
70 70 if SERVER_RUNNING_FILE:
71 71 with open(SERVER_RUNNING_FILE, 'wb') as f:
72 72 f.write(str(os.getpid()))
73 73 # register cleanup of that file when server exits
74 74 atexit.register(_remove_server_running_file)
75 75
76 76
77 77 class LazyWriter(object):
78 78 """
79 79 File-like object that opens a file lazily when it is first written
80 80 to.
81 81 """
82 82
83 83 def __init__(self, filename, mode='w'):
84 84 self.filename = filename
85 85 self.fileobj = None
86 86 self.lock = threading.Lock()
87 87 self.mode = mode
88 88
89 89 def open(self):
90 90 if self.fileobj is None:
91 91 with self.lock:
92 92 self.fileobj = open(self.filename, self.mode)
93 93 return self.fileobj
94 94
95 95 def close(self):
96 96 fileobj = self.fileobj
97 97 if fileobj is not None:
98 98 fileobj.close()
99 99
100 100 def __del__(self):
101 101 self.close()
102 102
103 103 def write(self, text):
104 104 fileobj = self.open()
105 105 fileobj.write(text)
106 106 fileobj.flush()
107 107
108 108 def writelines(self, text):
109 109 fileobj = self.open()
110 110 fileobj.writelines(text)
111 111 fileobj.flush()
112 112
113 113 def flush(self):
114 114 self.open().flush()
115 115
116 116
117 117 class Application(object):
118 118 """
119 119 Represents the vcs server application.
120 120
121 121 This object is responsible to initialize the application and all needed
122 122 libraries. After that it hooks together the different objects and provides
123 123 them a way to access things like configuration.
124 124 """
125 125
126 126 def __init__(
127 127 self, host, port=None, locale='', threadpool_size=None,
128 128 timeout=None, cache_config=None, remote_wsgi_=None):
129 129
130 130 self.host = host
131 131 self.port = int(port) or settings.PYRO_PORT
132 132 self.threadpool_size = (
133 133 int(threadpool_size) if threadpool_size else None)
134 134 self.locale = locale
135 135 self.timeout = timeout
136 136 self.cache_config = cache_config
137 137 self.remote_wsgi = remote_wsgi_ or remote_wsgi
138 138
139 139 def init(self):
140 140 """
141 141 Configure and hook together all relevant objects.
142 142 """
143 143 self._configure_locale()
144 144 self._configure_pyro()
145 145 self._initialize_cache()
146 146 self._create_daemon_and_remote_objects(host=self.host, port=self.port)
147 147
148 148 def run(self):
149 149 """
150 150 Start the main loop of the application.
151 151 """
152 152
153 153 if hasattr(os, 'getpid'):
154 154 log.info('Starting %s in PID %i.', __name__, os.getpid())
155 155 else:
156 156 log.info('Starting %s.', __name__)
157 157 if SERVER_RUNNING_FILE:
158 158 log.info('PID file written as %s', SERVER_RUNNING_FILE)
159 159 else:
160 160 log.info('No PID file written by default.')
161 161 when_ready(self)
162 162 try:
163 163 self._pyrodaemon.requestLoop(
164 164 loopCondition=lambda: not self._vcsserver._shutdown)
165 165 finally:
166 166 self._pyrodaemon.shutdown()
167 167
168 168 def _configure_locale(self):
169 169 if self.locale:
170 170 log.info('Settings locale: `LC_ALL` to %s' % self.locale)
171 171 else:
172 172 log.info(
173 173 'Configuring locale subsystem based on environment variables')
174 174
175 175 try:
176 176 # If self.locale is the empty string, then the locale
177 177 # module will use the environment variables. See the
178 178 # documentation of the package `locale`.
179 179 locale.setlocale(locale.LC_ALL, self.locale)
180 180
181 181 language_code, encoding = locale.getlocale()
182 182 log.info(
183 183 'Locale set to language code "%s" with encoding "%s".',
184 184 language_code, encoding)
185 185 except locale.Error:
186 186 log.exception(
187 187 'Cannot set locale, not configuring the locale system')
188 188
189 189 def _configure_pyro(self):
190 190 if self.threadpool_size is not None:
191 191 log.info("Threadpool size set to %s", self.threadpool_size)
192 192 Pyro4.config.THREADPOOL_SIZE = self.threadpool_size
193 193 if self.timeout not in (None, 0, 0.0, '0'):
194 194 log.info("Timeout for RPC calls set to %s seconds", self.timeout)
195 195 Pyro4.config.COMMTIMEOUT = float(self.timeout)
196 196 Pyro4.config.SERIALIZER = 'pickle'
197 197 Pyro4.config.SERIALIZERS_ACCEPTED.add('pickle')
198 198 Pyro4.config.SOCK_REUSE = True
199 199 # Uncomment the next line when you need to debug remote errors
200 200 # Pyro4.config.DETAILED_TRACEBACK = True
201 201
202 202 def _initialize_cache(self):
203 203 cache_config = parse_cache_config_options(self.cache_config)
204 204 log.info('Initializing beaker cache: %s' % cache_config)
205 205 self.cache = CacheManager(**cache_config)
206 206
207 207 def _create_daemon_and_remote_objects(self, host='localhost',
208 208 port=settings.PYRO_PORT):
209 209 daemon = Pyro4.Daemon(host=host, port=port)
210 210
211 211 self._vcsserver = VcsServer()
212 212 uri = daemon.register(
213 213 self._vcsserver, objectId=settings.PYRO_VCSSERVER)
214 214 log.info("Object registered = %s", uri)
215 215
216 216 if GitFactory and GitRemote:
217 217 git_repo_cache = self.cache.get_cache_region('git', region='repo_object')
218 218 git_factory = GitFactory(git_repo_cache)
219 219 self._git_remote = GitRemote(git_factory)
220 220 uri = daemon.register(self._git_remote, objectId=settings.PYRO_GIT)
221 221 log.info("Object registered = %s", uri)
222 222 else:
223 223 log.info("Git client import failed")
224 224
225 225 if MercurialFactory and HgRemote:
226 226 hg_repo_cache = self.cache.get_cache_region('hg', region='repo_object')
227 227 hg_factory = MercurialFactory(hg_repo_cache)
228 228 self._hg_remote = HgRemote(hg_factory)
229 229 uri = daemon.register(self._hg_remote, objectId=settings.PYRO_HG)
230 230 log.info("Object registered = %s", uri)
231 231 else:
232 232 log.info("Mercurial client import failed")
233 233
234 234 if SubversionFactory and SvnRemote:
235 235 svn_repo_cache = self.cache.get_cache_region('svn', region='repo_object')
236 236 svn_factory = SubversionFactory(svn_repo_cache)
237 237 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
238 238 uri = daemon.register(self._svn_remote, objectId=settings.PYRO_SVN)
239 239 log.info("Object registered = %s", uri)
240 240 else:
241 241 log.info("Subversion client import failed")
242 242
243 243 self._git_remote_wsgi = self.remote_wsgi.GitRemoteWsgi()
244 244 uri = daemon.register(self._git_remote_wsgi,
245 245 objectId=settings.PYRO_GIT_REMOTE_WSGI)
246 246 log.info("Object registered = %s", uri)
247 247
248 248 self._hg_remote_wsgi = self.remote_wsgi.HgRemoteWsgi()
249 249 uri = daemon.register(self._hg_remote_wsgi,
250 250 objectId=settings.PYRO_HG_REMOTE_WSGI)
251 251 log.info("Object registered = %s", uri)
252 252
253 253 self._pyrodaemon = daemon
254 254
255 255
256 256 class VcsServerCommand(object):
257 257
258 258 usage = '%prog'
259 259 description = """
260 260 Runs the VCS server
261 261 """
262 262 default_verbosity = 1
263 263
264 264 parser = optparse.OptionParser(
265 265 usage,
266 266 description=textwrap.dedent(description)
267 267 )
268 268 parser.add_option(
269 269 '--host',
270 270 type="str",
271 271 dest="host",
272 272 )
273 273 parser.add_option(
274 274 '--port',
275 275 type="int",
276 276 dest="port"
277 277 )
278 278 parser.add_option(
279 279 '--running-file',
280 280 dest='running_file',
281 281 metavar='RUNNING_FILE',
282 282 help="Create a running file after the server is initalized with "
283 283 "stored PID of process"
284 284 )
285 285 parser.add_option(
286 286 '--locale',
287 287 dest='locale',
288 288 help="Allows to set the locale, e.g. en_US.UTF-8",
289 289 default=""
290 290 )
291 291 parser.add_option(
292 292 '--log-file',
293 293 dest='log_file',
294 294 metavar='LOG_FILE',
295 295 help="Save output to the given log file (redirects stdout)"
296 296 )
297 297 parser.add_option(
298 298 '--log-level',
299 299 dest="log_level",
300 300 metavar="LOG_LEVEL",
301 301 help="use LOG_LEVEL to set log level "
302 302 "(debug,info,warning,error,critical)"
303 303 )
304 304 parser.add_option(
305 305 '--threadpool',
306 306 dest='threadpool_size',
307 307 type='int',
308 308 help="Set the size of the threadpool used to communicate with the "
309 309 "WSGI workers. This should be at least 6 times the number of "
310 310 "WSGI worker processes."
311 311 )
312 312 parser.add_option(
313 313 '--timeout',
314 314 dest='timeout',
315 315 type='float',
316 316 help="Set the timeout for RPC communication in seconds."
317 317 )
318 318 parser.add_option(
319 319 '--config',
320 320 dest='config_file',
321 321 type='string',
322 322 help="Configuration file for vcsserver."
323 323 )
324 324
325 325 def __init__(self, argv, quiet=False):
326 326 self.options, self.args = self.parser.parse_args(argv[1:])
327 327 if quiet:
328 328 self.options.verbose = 0
329 329
330 330 def _get_file_config(self):
331 331 ini_conf = {}
332 332 conf = configobj.ConfigObj(self.options.config_file)
333 333 if 'DEFAULT' in conf:
334 334 ini_conf = conf['DEFAULT']
335 335
336 336 return ini_conf
337 337
338 338 def _show_config(self, vcsserver_config):
339 339 order = [
340 340 'config_file',
341 341 'host',
342 342 'port',
343 343 'log_file',
344 344 'log_level',
345 345 'locale',
346 346 'threadpool_size',
347 347 'timeout',
348 348 'cache_config',
349 349 ]
350 350
351 351 def sorter(k):
352 352 return dict([(y, x) for x, y in enumerate(order)]).get(k)
353 353
354 354 _config = []
355 355 for k in sorted(vcsserver_config.keys(), key=sorter):
356 356 v = vcsserver_config[k]
357 357 # construct padded key for display eg %-20s % = key: val
358 358 k_formatted = ('%-'+str(len(max(order, key=len))+1)+'s') % (k+':')
359 359 _config.append(' * %s %s' % (k_formatted, v))
360 360 log.info('\n[vcsserver configuration]:\n'+'\n'.join(_config))
361 361
362 362 def _get_vcsserver_configuration(self):
363 363 _defaults = {
364 364 'config_file': None,
365 365 'git_path': 'git',
366 366 'host': 'localhost',
367 367 'port': settings.PYRO_PORT,
368 368 'log_file': None,
369 369 'log_level': 'debug',
370 370 'locale': None,
371 371 'threadpool_size': 16,
372 372 'timeout': None,
373 373
374 374 # Development support
375 375 'dev.use_echo_app': False,
376 376
377 377 # caches, baker style config
378 378 'beaker.cache.regions': 'repo_object',
379 379 'beaker.cache.repo_object.expire': '10',
380 380 'beaker.cache.repo_object.type': 'memory',
381 381 }
382 382 config = {}
383 383 config.update(_defaults)
384 384 # overwrite defaults with one loaded from file
385 385 config.update(self._get_file_config())
386 386
387 387 # overwrite with self.option which has the top priority
388 388 for k, v in self.options.__dict__.items():
389 389 if v or v == 0:
390 390 config[k] = v
391 391
392 392 # clear all "extra" keys if they are somehow passed,
393 393 # we only want defaults, so any extra stuff from self.options is cleared
394 394 # except beaker stuff which needs to be dynamic
395 395 for k in [k for k in config.copy().keys() if not k.startswith('beaker.cache.')]:
396 396 if k not in _defaults:
397 397 del config[k]
398 398
399 399 # group together the cache into one key.
400 400 # Needed further for beaker lib configuration
401 401 _k = {}
402 402 for k in [k for k in config.copy() if k.startswith('beaker.cache.')]:
403 403 _k[k] = config.pop(k)
404 404 config['cache_config'] = _k
405 405
406 406 return config
407 407
408 408 def out(self, msg): # pragma: no cover
409 409 if self.options.verbose > 0:
410 410 print(msg)
411 411
412 412 def run(self): # pragma: no cover
413 413 vcsserver_config = self._get_vcsserver_configuration()
414 414
415 415 # Ensure the log file is writeable
416 416 if vcsserver_config['log_file']:
417 417 stdout_log = self._configure_logfile()
418 418 else:
419 419 stdout_log = None
420 420
421 421 # set PID file with running lock
422 422 if self.options.running_file:
423 423 global SERVER_RUNNING_FILE
424 424 SERVER_RUNNING_FILE = self.options.running_file
425 425
426 426 # configure logging, and logging based on configuration file
427 427 self._configure_logging(level=vcsserver_config['log_level'],
428 428 stream=stdout_log)
429 429 if self.options.config_file:
430 430 if not os.path.isfile(self.options.config_file):
431 431 raise OSError('File %s does not exist' %
432 432 self.options.config_file)
433 433
434 434 self._configure_file_logging(self.options.config_file)
435 435
436 436 self._configure_settings(vcsserver_config)
437 437
438 438 # display current configuration of vcsserver
439 439 self._show_config(vcsserver_config)
440 440
441 441 if not vcsserver_config['dev.use_echo_app']:
442 442 remote_wsgi_mod = remote_wsgi
443 443 else:
444 444 log.warning("Using EchoApp for VCS endpoints.")
445 445 remote_wsgi_mod = remote_wsgi_stub
446 446
447 447 app = Application(
448 448 host=vcsserver_config['host'],
449 449 port=vcsserver_config['port'],
450 450 locale=vcsserver_config['locale'],
451 451 threadpool_size=vcsserver_config['threadpool_size'],
452 452 timeout=vcsserver_config['timeout'],
453 453 cache_config=vcsserver_config['cache_config'],
454 454 remote_wsgi_=remote_wsgi_mod)
455 455 app.init()
456 456 app.run()
457 457
458 458 def _configure_logging(self, level, stream=None):
459 459 _format = (
460 460 '%(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s')
461 461 levels = {
462 462 'debug': logging.DEBUG,
463 463 'info': logging.INFO,
464 464 'warning': logging.WARNING,
465 465 'error': logging.ERROR,
466 466 'critical': logging.CRITICAL,
467 467 }
468 468 try:
469 469 level = levels[level]
470 470 except KeyError:
471 471 raise AttributeError(
472 472 'Invalid log level please use one of %s' % (levels.keys(),))
473 473 logging.basicConfig(format=_format, stream=stream, level=level)
474 474 logging.getLogger('Pyro4').setLevel(level)
475 475
476 476 def _configure_file_logging(self, config):
477 477 import logging.config
478 478 try:
479 479 logging.config.fileConfig(config)
480 480 except Exception as e:
481 481 log.warning('Failed to configure logging based on given '
482 482 'config file. Error: %s' % e)
483 483
484 484 def _configure_logfile(self):
485 485 try:
486 486 writeable_log_file = open(self.options.log_file, 'a')
487 487 except IOError as ioe:
488 488 msg = 'Error: Unable to write to log file: %s' % ioe
489 489 raise ValueError(msg)
490 490 writeable_log_file.close()
491 491 stdout_log = LazyWriter(self.options.log_file, 'a')
492 492 sys.stdout = stdout_log
493 493 sys.stderr = stdout_log
494 494 return stdout_log
495 495
496 496 def _configure_settings(self, config):
497 497 """
498 498 Configure the settings module based on the given `config`.
499 499 """
500 500 settings.GIT_EXECUTABLE = config['git_path']
501 501
502 502
503 503 def main(argv=sys.argv, quiet=False):
504 504 if MercurialFactory:
505 505 hgpatches.patch_largefiles_capabilities()
506 506 hgpatches.patch_subrepo_type_mapping()
507 507 command = VcsServerCommand(argv, quiet=quiet)
508 508 return command.run()
@@ -1,375 +1,375 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 """Handles the Git smart protocol."""
19 19
20 20 import os
21 21 import socket
22 22 import logging
23 23
24 24 import simplejson as json
25 25 import dulwich.protocol
26 26 from webob import Request, Response, exc
27 27
28 28 from vcsserver import hooks, subprocessio
29 29
30 30
31 31 log = logging.getLogger(__name__)
32 32
33 33
34 34 class FileWrapper(object):
35 35 """File wrapper that ensures how much data is read from it."""
36 36
37 37 def __init__(self, fd, content_length):
38 38 self.fd = fd
39 39 self.content_length = content_length
40 40 self.remain = content_length
41 41
42 42 def read(self, size):
43 43 if size <= self.remain:
44 44 try:
45 45 data = self.fd.read(size)
46 46 except socket.error:
47 47 raise IOError(self)
48 48 self.remain -= size
49 49 elif self.remain:
50 50 data = self.fd.read(self.remain)
51 51 self.remain = 0
52 52 else:
53 53 data = None
54 54 return data
55 55
56 56 def __repr__(self):
57 57 return '<FileWrapper %s len: %s, read: %s>' % (
58 58 self.fd, self.content_length, self.content_length - self.remain
59 59 )
60 60
61 61
62 62 class GitRepository(object):
63 63 """WSGI app for handling Git smart protocol endpoints."""
64 64
65 65 git_folder_signature = frozenset(
66 66 ('config', 'head', 'info', 'objects', 'refs'))
67 67 commands = frozenset(('git-upload-pack', 'git-receive-pack'))
68 68 valid_accepts = frozenset(('application/x-%s-result' %
69 69 c for c in commands))
70 70
71 71 # The last bytes are the SHA1 of the first 12 bytes.
72 72 EMPTY_PACK = (
73 73 'PACK\x00\x00\x00\x02\x00\x00\x00\x00' +
74 74 '\x02\x9d\x08\x82;\xd8\xa8\xea\xb5\x10\xadj\xc7\\\x82<\xfd>\xd3\x1e'
75 75 )
76 76 SIDE_BAND_CAPS = frozenset(('side-band', 'side-band-64k'))
77 77
78 78 def __init__(self, repo_name, content_path, git_path, update_server_info,
79 79 extras):
80 80 files = frozenset(f.lower() for f in os.listdir(content_path))
81 81 valid_dir_signature = self.git_folder_signature.issubset(files)
82 82
83 83 if not valid_dir_signature:
84 84 raise OSError('%s missing git signature' % content_path)
85 85
86 86 self.content_path = content_path
87 87 self.repo_name = repo_name
88 88 self.extras = extras
89 89 self.git_path = git_path
90 90 self.update_server_info = update_server_info
91 91
92 92 def _get_fixedpath(self, path):
93 93 """
94 94 Small fix for repo_path
95 95
96 96 :param path:
97 97 """
98 98 return path.split(self.repo_name, 1)[-1].strip('/')
99 99
100 100 def inforefs(self, request, unused_environ):
101 101 """
102 102 WSGI Response producer for HTTP GET Git Smart
103 103 HTTP /info/refs request.
104 104 """
105 105
106 106 git_command = request.GET.get('service')
107 107 if git_command not in self.commands:
108 108 log.debug('command %s not allowed', git_command)
109 109 return exc.HTTPForbidden()
110 110
111 111 # please, resist the urge to add '\n' to git capture and increment
112 112 # line count by 1.
113 113 # by git docs: Documentation/technical/http-protocol.txt#L214 \n is
114 114 # a part of protocol.
115 115 # The code in Git client not only does NOT need '\n', but actually
116 116 # blows up if you sprinkle "flush" (0000) as "0001\n".
117 117 # It reads binary, per number of bytes specified.
118 118 # if you do add '\n' as part of data, count it.
119 119 server_advert = '# service=%s\n' % git_command
120 120 packet_len = str(hex(len(server_advert) + 4)[2:].rjust(4, '0')).lower()
121 121 try:
122 122 gitenv = dict(os.environ)
123 123 # forget all configs
124 124 gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
125 125 command = [self.git_path, git_command[4:], '--stateless-rpc',
126 126 '--advertise-refs', self.content_path]
127 127 out = subprocessio.SubprocessIOChunker(
128 128 command,
129 129 env=gitenv,
130 130 starting_values=[packet_len + server_advert + '0000'],
131 131 shell=False
132 132 )
133 133 except EnvironmentError:
134 134 log.exception('Error processing command')
135 135 raise exc.HTTPExpectationFailed()
136 136
137 137 resp = Response()
138 138 resp.content_type = 'application/x-%s-advertisement' % str(git_command)
139 139 resp.charset = None
140 140 resp.app_iter = out
141 141
142 142 return resp
143 143
144 144 def _get_want_capabilities(self, request):
145 145 """Read the capabilities found in the first want line of the request."""
146 146 pos = request.body_file_seekable.tell()
147 147 first_line = request.body_file_seekable.readline()
148 148 request.body_file_seekable.seek(pos)
149 149
150 150 return frozenset(
151 151 dulwich.protocol.extract_want_line_capabilities(first_line)[1])
152 152
153 153 def _build_failed_pre_pull_response(self, capabilities, pre_pull_messages):
154 154 """
155 155 Construct a response with an empty PACK file.
156 156
157 157 We use an empty PACK file, as that would trigger the failure of the pull
158 158 or clone command.
159 159
160 160 We also print in the error output a message explaining why the command
161 161 was aborted.
162 162
163 163 If aditionally, the user is accepting messages we send them the output
164 164 of the pre-pull hook.
165 165
166 166 Note that for clients not supporting side-band we just send them the
167 167 emtpy PACK file.
168 168 """
169 169 if self.SIDE_BAND_CAPS.intersection(capabilities):
170 170 response = []
171 171 proto = dulwich.protocol.Protocol(None, response.append)
172 172 proto.write_pkt_line('NAK\n')
173 173 self._write_sideband_to_proto(pre_pull_messages, proto,
174 174 capabilities)
175 175 # N.B.(skreft): Do not change the sideband channel to 3, as that
176 176 # produces a fatal error in the client:
177 177 # fatal: error in sideband demultiplexer
178 178 proto.write_sideband(2, 'Pre pull hook failed: aborting\n')
179 179 proto.write_sideband(1, self.EMPTY_PACK)
180 180
181 181 # writes 0000
182 182 proto.write_pkt_line(None)
183 183
184 184 return response
185 185 else:
186 186 return [self.EMPTY_PACK]
187 187
188 188 def _write_sideband_to_proto(self, data, proto, capabilities):
189 189 """
190 190 Write the data to the proto's sideband number 2.
191 191
192 192 We do not use dulwich's write_sideband directly as it only supports
193 193 side-band-64k.
194 194 """
195 195 if not data:
196 196 return
197 197
198 198 # N.B.(skreft): The values below are explained in the pack protocol
199 199 # documentation, section Packfile Data.
200 200 # https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt
201 201 if 'side-band-64k' in capabilities:
202 202 chunk_size = 65515
203 203 elif 'side-band' in capabilities:
204 204 chunk_size = 995
205 205 else:
206 206 return
207 207
208 208 chunker = (
209 209 data[i:i + chunk_size] for i in xrange(0, len(data), chunk_size))
210 210
211 211 for chunk in chunker:
212 212 proto.write_sideband(2, chunk)
213 213
214 214 def _get_messages(self, data, capabilities):
215 215 """Return a list with packets for sending data in sideband number 2."""
216 216 response = []
217 217 proto = dulwich.protocol.Protocol(None, response.append)
218 218
219 219 self._write_sideband_to_proto(data, proto, capabilities)
220 220
221 221 return response
222 222
223 223 def _inject_messages_to_response(self, response, capabilities,
224 224 start_messages, end_messages):
225 225 """
226 226 Given a list reponse we inject the pre/post-pull messages.
227 227
228 228 We only inject the messages if the client supports sideband, and the
229 229 response has the format:
230 230 0008NAK\n...0000
231 231
232 232 Note that we do not check the no-progress capability as by default, git
233 233 sends it, which effectively would block all messages.
234 234 """
235 235 if not self.SIDE_BAND_CAPS.intersection(capabilities):
236 236 return response
237 237
238 238 if (not response[0].startswith('0008NAK\n') or
239 239 not response[-1].endswith('0000')):
240 240 return response
241 241
242 242 if not start_messages and not end_messages:
243 243 return response
244 244
245 245 new_response = ['0008NAK\n']
246 246 new_response.extend(self._get_messages(start_messages, capabilities))
247 247 if len(response) == 1:
248 248 new_response.append(response[0][8:-4])
249 249 else:
250 250 new_response.append(response[0][8:])
251 251 new_response.extend(response[1:-1])
252 252 new_response.append(response[-1][:-4])
253 253 new_response.extend(self._get_messages(end_messages, capabilities))
254 254 new_response.append('0000')
255 255
256 256 return new_response
257 257
258 258 def backend(self, request, environ):
259 259 """
260 260 WSGI Response producer for HTTP POST Git Smart HTTP requests.
261 261 Reads commands and data from HTTP POST's body.
262 262 returns an iterator obj with contents of git command's
263 263 response to stdout
264 264 """
265 265 # TODO(skreft): think how we could detect an HTTPLockedException, as
266 266 # we probably want to have the same mechanism used by mercurial and
267 267 # simplevcs.
268 268 # For that we would need to parse the output of the command looking for
269 269 # some signs of the HTTPLockedError, parse the data and reraise it in
270 270 # pygrack. However, that would interfere with the streaming.
271 271 #
272 272 # Now the output of a blocked push is:
273 273 # Pushing to http://test_regular:test12@127.0.0.1:5001/vcs_test_git
274 274 # POST git-receive-pack (1047 bytes)
275 275 # remote: ERROR: Repository `vcs_test_git` locked by user `test_admin`. Reason:`lock_auto`
276 276 # To http://test_regular:test12@127.0.0.1:5001/vcs_test_git
277 277 # ! [remote rejected] master -> master (pre-receive hook declined)
278 278 # error: failed to push some refs to 'http://test_regular:test12@127.0.0.1:5001/vcs_test_git'
279 279
280 280 git_command = self._get_fixedpath(request.path_info)
281 281 if git_command not in self.commands:
282 282 log.debug('command %s not allowed', git_command)
283 283 return exc.HTTPForbidden()
284 284
285 285 capabilities = None
286 286 if git_command == 'git-upload-pack':
287 287 capabilities = self._get_want_capabilities(request)
288 288
289 289 if 'CONTENT_LENGTH' in environ:
290 290 inputstream = FileWrapper(request.body_file_seekable,
291 291 request.content_length)
292 292 else:
293 293 inputstream = request.body_file_seekable
294 294
295 295 resp = Response()
296 296 resp.content_type = ('application/x-%s-result' %
297 297 git_command.encode('utf8'))
298 298 resp.charset = None
299 299
300 300 if git_command == 'git-upload-pack':
301 301 status, pre_pull_messages = hooks.git_pre_pull(self.extras)
302 302 if status != 0:
303 303 resp.app_iter = self._build_failed_pre_pull_response(
304 304 capabilities, pre_pull_messages)
305 305 return resp
306 306
307 307 gitenv = dict(os.environ)
308 308 # forget all configs
309 309 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
310 310 gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
311 311 cmd = [self.git_path, git_command[4:], '--stateless-rpc',
312 312 self.content_path]
313 313 log.debug('handling cmd %s', cmd)
314 314
315 315 out = subprocessio.SubprocessIOChunker(
316 316 cmd,
317 317 inputstream=inputstream,
318 318 env=gitenv,
319 319 cwd=self.content_path,
320 320 shell=False,
321 321 fail_on_stderr=False,
322 322 fail_on_return_code=False
323 323 )
324 324
325 325 if self.update_server_info and git_command == 'git-receive-pack':
326 326 # We need to fully consume the iterator here, as the
327 327 # update-server-info command needs to be run after the push.
328 328 out = list(out)
329 329
330 330 # Updating refs manually after each push.
331 331 # This is required as some clients are exposing Git repos internally
332 332 # with the dumb protocol.
333 333 cmd = [self.git_path, 'update-server-info']
334 334 log.debug('handling cmd %s', cmd)
335 335 output = subprocessio.SubprocessIOChunker(
336 336 cmd,
337 337 inputstream=inputstream,
338 338 env=gitenv,
339 339 cwd=self.content_path,
340 340 shell=False,
341 341 fail_on_stderr=False,
342 342 fail_on_return_code=False
343 343 )
344 344 # Consume all the output so the subprocess finishes
345 345 for _ in output:
346 346 pass
347 347
348 348 if git_command == 'git-upload-pack':
349 349 out = list(out)
350 350 unused_status, post_pull_messages = hooks.git_post_pull(self.extras)
351 351 resp.app_iter = self._inject_messages_to_response(
352 352 out, capabilities, pre_pull_messages, post_pull_messages)
353 353 else:
354 354 resp.app_iter = out
355 355
356 356 return resp
357 357
358 358 def __call__(self, environ, start_response):
359 359 request = Request(environ)
360 360 _path = self._get_fixedpath(request.path_info)
361 361 if _path.startswith('info/refs'):
362 362 app = self.inforefs
363 363 else:
364 364 app = self.backend
365 365
366 366 try:
367 367 resp = app(request, environ)
368 368 except exc.HTTPException as error:
369 369 log.exception('HTTP Error')
370 370 resp = error
371 371 except Exception:
372 372 log.exception('Unknown error')
373 373 resp = exc.HTTPInternalServerError()
374 374
375 375 return resp(environ, start_response)
@@ -1,34 +1,34 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 from vcsserver import scm_app, wsgi_app_caller
19 19
20 20
21 21 class GitRemoteWsgi(object):
22 22 def handle(self, environ, input_data, *args, **kwargs):
23 23 app = wsgi_app_caller.WSGIAppCaller(
24 24 scm_app.create_git_wsgi_app(*args, **kwargs))
25 25
26 26 return app.handle(environ, input_data)
27 27
28 28
29 29 class HgRemoteWsgi(object):
30 30 def handle(self, environ, input_data, *args, **kwargs):
31 31 app = wsgi_app_caller.WSGIAppCaller(
32 32 scm_app.create_hg_wsgi_app(*args, **kwargs))
33 33
34 34 return app.handle(environ, input_data)
@@ -1,174 +1,174 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import logging
19 19 import os
20 20
21 21 import mercurial
22 22 import mercurial.error
23 23 import mercurial.hgweb.common
24 24 import mercurial.hgweb.hgweb_mod
25 25 import mercurial.hgweb.protocol
26 26 import webob.exc
27 27
28 28 from vcsserver import pygrack, exceptions, settings
29 29
30 30
31 31 log = logging.getLogger(__name__)
32 32
33 33
34 34 # propagated from mercurial documentation
35 35 HG_UI_SECTIONS = [
36 36 'alias', 'auth', 'decode/encode', 'defaults', 'diff', 'email', 'extensions',
37 37 'format', 'merge-patterns', 'merge-tools', 'hooks', 'http_proxy', 'smtp',
38 38 'patch', 'paths', 'profiling', 'server', 'trusted', 'ui', 'web',
39 39 ]
40 40
41 41
42 42 class HgWeb(mercurial.hgweb.hgweb_mod.hgweb):
43 43 """Extension of hgweb that simplifies some functions."""
44 44
45 45 def _get_view(self, repo):
46 46 """Views are not supported."""
47 47 return repo
48 48
49 49 def loadsubweb(self):
50 50 """The result is only used in the templater method which is not used."""
51 51 return None
52 52
53 53 def run(self):
54 54 """Unused function so raise an exception if accidentally called."""
55 55 raise NotImplementedError
56 56
57 57 def templater(self, req):
58 58 """Function used in an unreachable code path.
59 59
60 60 This code is unreachable because we guarantee that the HTTP request,
61 61 corresponds to a Mercurial command. See the is_hg method. So, we are
62 62 never going to get a user-visible url.
63 63 """
64 64 raise NotImplementedError
65 65
66 66 def archivelist(self, nodeid):
67 67 """Unused function so raise an exception if accidentally called."""
68 68 raise NotImplementedError
69 69
70 70 def run_wsgi(self, req):
71 71 """Check the request has a valid command, failing fast otherwise."""
72 72 cmd = req.form.get('cmd', [''])[0]
73 73 if not mercurial.hgweb.protocol.iscmd(cmd):
74 74 req.respond(
75 75 mercurial.hgweb.common.ErrorResponse(
76 76 mercurial.hgweb.common.HTTP_BAD_REQUEST),
77 77 mercurial.hgweb.protocol.HGTYPE
78 78 )
79 79 return ['']
80 80
81 81 return super(HgWeb, self).run_wsgi(req)
82 82
83 83
84 84 def make_hg_ui_from_config(repo_config):
85 85 baseui = mercurial.ui.ui()
86 86
87 87 # clean the baseui object
88 88 baseui._ocfg = mercurial.config.config()
89 89 baseui._ucfg = mercurial.config.config()
90 90 baseui._tcfg = mercurial.config.config()
91 91
92 92 for section, option, value in repo_config:
93 93 baseui.setconfig(section, option, value)
94 94
95 95 # make our hgweb quiet so it doesn't print output
96 96 baseui.setconfig('ui', 'quiet', 'true')
97 97
98 98 return baseui
99 99
100 100
101 101 def update_hg_ui_from_hgrc(baseui, repo_path):
102 102 path = os.path.join(repo_path, '.hg', 'hgrc')
103 103
104 104 if not os.path.isfile(path):
105 105 log.debug('hgrc file is not present at %s, skipping...', path)
106 106 return
107 107 log.debug('reading hgrc from %s', path)
108 108 cfg = mercurial.config.config()
109 109 cfg.read(path)
110 110 for section in HG_UI_SECTIONS:
111 111 for k, v in cfg.items(section):
112 112 log.debug('settings ui from file: [%s] %s=%s', section, k, v)
113 113 baseui.setconfig(section, k, v)
114 114
115 115
116 116 def create_hg_wsgi_app(repo_path, repo_name, config):
117 117 """
118 118 Prepares a WSGI application to handle Mercurial requests.
119 119
120 120 :param config: is a list of 3-item tuples representing a ConfigObject
121 121 (it is the serialized version of the config object).
122 122 """
123 123 log.debug("Creating Mercurial WSGI application")
124 124
125 125 baseui = make_hg_ui_from_config(config)
126 126 update_hg_ui_from_hgrc(baseui, repo_path)
127 127
128 128 try:
129 129 return HgWeb(repo_path, name=repo_name, baseui=baseui)
130 130 except mercurial.error.RequirementError as exc:
131 131 raise exceptions.RequirementException(exc)
132 132
133 133
134 134 class GitHandler(object):
135 135 def __init__(self, repo_location, repo_name, git_path, update_server_info,
136 136 extras):
137 137 if not os.path.isdir(repo_location):
138 138 raise OSError(repo_location)
139 139 self.content_path = repo_location
140 140 self.repo_name = repo_name
141 141 self.repo_location = repo_location
142 142 self.extras = extras
143 143 self.git_path = git_path
144 144 self.update_server_info = update_server_info
145 145
146 146 def __call__(self, environ, start_response):
147 147 app = webob.exc.HTTPNotFound()
148 148 candidate_paths = (
149 149 self.content_path, os.path.join(self.content_path, '.git'))
150 150
151 151 for content_path in candidate_paths:
152 152 try:
153 153 app = pygrack.GitRepository(
154 154 self.repo_name, content_path, self.git_path,
155 155 self.update_server_info, self.extras)
156 156 break
157 157 except OSError:
158 158 continue
159 159
160 160 return app(environ, start_response)
161 161
162 162
163 163 def create_git_wsgi_app(repo_path, repo_name, config):
164 164 """
165 165 Creates a WSGI application to handle Git requests.
166 166
167 167 :param config: is a dictionary holding the extras.
168 168 """
169 169 git_path = settings.GIT_EXECUTABLE
170 170 update_server_info = config.pop('git_update_server_info')
171 171 app = GitHandler(
172 172 repo_path, repo_name, git_path, update_server_info, config)
173 173
174 174 return app
@@ -1,78 +1,78 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import gc
19 19 import logging
20 20 import os
21 21 import time
22 22
23 23
24 24 log = logging.getLogger(__name__)
25 25
26 26
27 27 class VcsServer(object):
28 28 """
29 29 Exposed remote interface of the vcsserver itself.
30 30
31 31 This object can be used to manage the server remotely. Right now the main
32 32 use case is to allow to shut down the server.
33 33 """
34 34
35 35 _shutdown = False
36 36
37 37 def shutdown(self):
38 38 self._shutdown = True
39 39
40 40 def ping(self):
41 41 """
42 42 Utility to probe a server connection.
43 43 """
44 44 log.debug("Received server ping.")
45 45
46 46 def echo(self, data):
47 47 """
48 48 Utility for performance testing.
49 49
50 50 Allows to pass in arbitrary data and will return this data.
51 51 """
52 52 log.debug("Received server echo.")
53 53 return data
54 54
55 55 def sleep(self, seconds):
56 56 """
57 57 Utility to simulate long running server interaction.
58 58 """
59 59 log.debug("Sleeping %s seconds", seconds)
60 60 time.sleep(seconds)
61 61
62 62 def get_pid(self):
63 63 """
64 64 Allows to discover the PID based on a proxy object.
65 65 """
66 66 return os.getpid()
67 67
68 68 def run_gc(self):
69 69 """
70 70 Allows to trigger the garbage collector.
71 71
72 72 Main intention is to support statistics gathering during test runs.
73 73 """
74 74 freed_objects = gc.collect()
75 75 return {
76 76 'freed_objects': freed_objects,
77 77 'garbage': len(gc.garbage),
78 78 }
@@ -1,30 +1,30 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18
19 19 PYRO_PORT = 9900
20 20
21 21 PYRO_GIT = 'git_remote'
22 22 PYRO_HG = 'hg_remote'
23 23 PYRO_SVN = 'svn_remote'
24 24 PYRO_VCSSERVER = 'vcs_server'
25 25 PYRO_GIT_REMOTE_WSGI = 'git_remote_wsgi'
26 26 PYRO_HG_REMOTE_WSGI = 'hg_remote_wsgi'
27 27
28 28 WIRE_ENCODING = 'UTF-8'
29 29
30 30 GIT_EXECUTABLE = 'git'
@@ -1,627 +1,651 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 from __future__ import absolute_import
19 19
20 20 from urllib2 import URLError
21 21 import logging
22 22 import posixpath as vcspath
23 23 import StringIO
24 24 import subprocess
25 25 import urllib
26 26
27 27 import svn.client
28 28 import svn.core
29 29 import svn.delta
30 30 import svn.diff
31 31 import svn.fs
32 32 import svn.repos
33 33
34 34 from vcsserver import svn_diff
35 35 from vcsserver import exceptions
36 36 from vcsserver.base import RepoFactory
37 37
38 38
39 39 log = logging.getLogger(__name__)
40 40
41 41
42 42 # Set of svn compatible version flags.
43 43 # Compare with subversion/svnadmin/svnadmin.c
44 44 svn_compatible_versions = set([
45 45 'pre-1.4-compatible',
46 46 'pre-1.5-compatible',
47 47 'pre-1.6-compatible',
48 48 'pre-1.8-compatible',
49 49 ])
50 50
51 51
52 52 def reraise_safe_exceptions(func):
53 53 """Decorator for converting svn exceptions to something neutral."""
54 54 def wrapper(*args, **kwargs):
55 55 try:
56 56 return func(*args, **kwargs)
57 57 except Exception as e:
58 58 if not hasattr(e, '_vcs_kind'):
59 59 log.exception("Unhandled exception in hg remote call")
60 60 raise_from_original(exceptions.UnhandledException)
61 61 raise
62 62 return wrapper
63 63
64 64
65 65 def raise_from_original(new_type):
66 66 """
67 67 Raise a new exception type with original args and traceback.
68 68 """
69 69 _, original, traceback = sys.exc_info()
70 70 try:
71 71 raise new_type(*original.args), None, traceback
72 72 finally:
73 73 del traceback
74 74
75 75
76 76 class SubversionFactory(RepoFactory):
77 77
78 78 def _create_repo(self, wire, create, compatible_version):
79 79 path = svn.core.svn_path_canonicalize(wire['path'])
80 80 if create:
81 81 fs_config = {}
82 82 if compatible_version:
83 83 if compatible_version not in svn_compatible_versions:
84 84 raise Exception('Unknown SVN compatible version "{}"'
85 85 .format(compatible_version))
86 86 log.debug('Create SVN repo with compatible version "%s"',
87 87 compatible_version)
88 88 fs_config[compatible_version] = '1'
89 89 repo = svn.repos.create(path, "", "", None, fs_config)
90 90 else:
91 91 repo = svn.repos.open(path)
92 92 return repo
93 93
94 94 def repo(self, wire, create=False, compatible_version=None):
95 95 def create_new_repo():
96 96 return self._create_repo(wire, create, compatible_version)
97 97
98 98 return self._repo(wire, create_new_repo)
99 99
100 100
101 101
102 102 NODE_TYPE_MAPPING = {
103 103 svn.core.svn_node_file: 'file',
104 104 svn.core.svn_node_dir: 'dir',
105 105 }
106 106
107 107
108 108 class SvnRemote(object):
109 109
110 110 def __init__(self, factory, hg_factory=None):
111 111 self._factory = factory
112 112 # TODO: Remove once we do not use internal Mercurial objects anymore
113 113 # for subversion
114 114 self._hg_factory = hg_factory
115 115
116 116 @reraise_safe_exceptions
117 117 def discover_svn_version(self):
118 118 try:
119 119 import svn.core
120 120 svn_ver = svn.core.SVN_VERSION
121 121 except ImportError:
122 122 svn_ver = None
123 123 return svn_ver
124 124
125 125 def check_url(self, url, config_items):
126 126 # this can throw exception if not installed, but we detect this
127 127 from hgsubversion import svnrepo
128 128
129 129 baseui = self._hg_factory._create_config(config_items)
130 130 # uuid function get's only valid UUID from proper repo, else
131 131 # throws exception
132 132 try:
133 133 svnrepo.svnremoterepo(baseui, url).svn.uuid
134 134 except:
135 135 log.debug("Invalid svn url: %s", url)
136 136 raise URLError(
137 137 '"%s" is not a valid Subversion source url.' % (url, ))
138 138 return True
139 139
140 140 def is_path_valid_repository(self, wire, path):
141 141 try:
142 142 svn.repos.open(path)
143 143 except svn.core.SubversionException:
144 144 log.debug("Invalid Subversion path %s", path)
145 145 return False
146 146 return True
147 147
148 148 def lookup(self, wire, revision):
149 149 if revision not in [-1, None, 'HEAD']:
150 150 raise NotImplementedError
151 151 repo = self._factory.repo(wire)
152 152 fs_ptr = svn.repos.fs(repo)
153 153 head = svn.fs.youngest_rev(fs_ptr)
154 154 return head
155 155
156 156 def lookup_interval(self, wire, start_ts, end_ts):
157 157 repo = self._factory.repo(wire)
158 158 fsobj = svn.repos.fs(repo)
159 159 start_rev = None
160 160 end_rev = None
161 161 if start_ts:
162 162 start_ts_svn = apr_time_t(start_ts)
163 163 start_rev = svn.repos.dated_revision(repo, start_ts_svn) + 1
164 164 else:
165 165 start_rev = 1
166 166 if end_ts:
167 167 end_ts_svn = apr_time_t(end_ts)
168 168 end_rev = svn.repos.dated_revision(repo, end_ts_svn)
169 169 else:
170 170 end_rev = svn.fs.youngest_rev(fsobj)
171 171 return start_rev, end_rev
172 172
173 173 def revision_properties(self, wire, revision):
174 174 repo = self._factory.repo(wire)
175 175 fs_ptr = svn.repos.fs(repo)
176 176 return svn.fs.revision_proplist(fs_ptr, revision)
177 177
178 178 def revision_changes(self, wire, revision):
179 179
180 180 repo = self._factory.repo(wire)
181 181 fsobj = svn.repos.fs(repo)
182 182 rev_root = svn.fs.revision_root(fsobj, revision)
183 183
184 184 editor = svn.repos.ChangeCollector(fsobj, rev_root)
185 185 editor_ptr, editor_baton = svn.delta.make_editor(editor)
186 186 base_dir = ""
187 187 send_deltas = False
188 188 svn.repos.replay2(
189 189 rev_root, base_dir, svn.core.SVN_INVALID_REVNUM, send_deltas,
190 190 editor_ptr, editor_baton, None)
191 191
192 192 added = []
193 193 changed = []
194 194 removed = []
195 195
196 196 # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
197 197 for path, change in editor.changes.iteritems():
198 198 # TODO: Decide what to do with directory nodes. Subversion can add
199 199 # empty directories.
200 200
201 201 if change.item_kind == svn.core.svn_node_dir:
202 202 continue
203 203 if change.action in [svn.repos.CHANGE_ACTION_ADD]:
204 204 added.append(path)
205 205 elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
206 206 svn.repos.CHANGE_ACTION_REPLACE]:
207 207 changed.append(path)
208 208 elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
209 209 removed.append(path)
210 210 else:
211 211 raise NotImplementedError(
212 212 "Action %s not supported on path %s" % (
213 213 change.action, path))
214 214
215 215 changes = {
216 216 'added': added,
217 217 'changed': changed,
218 218 'removed': removed,
219 219 }
220 220 return changes
221 221
222 222 def node_history(self, wire, path, revision, limit):
223 223 cross_copies = False
224 224 repo = self._factory.repo(wire)
225 225 fsobj = svn.repos.fs(repo)
226 226 rev_root = svn.fs.revision_root(fsobj, revision)
227 227
228 228 history_revisions = []
229 229 history = svn.fs.node_history(rev_root, path)
230 230 history = svn.fs.history_prev(history, cross_copies)
231 231 while history:
232 232 __, node_revision = svn.fs.history_location(history)
233 233 history_revisions.append(node_revision)
234 234 if limit and len(history_revisions) >= limit:
235 235 break
236 236 history = svn.fs.history_prev(history, cross_copies)
237 237 return history_revisions
238 238
239 239 def node_properties(self, wire, path, revision):
240 240 repo = self._factory.repo(wire)
241 241 fsobj = svn.repos.fs(repo)
242 242 rev_root = svn.fs.revision_root(fsobj, revision)
243 243 return svn.fs.node_proplist(rev_root, path)
244 244
245 245 def file_annotate(self, wire, path, revision):
246 246 abs_path = 'file://' + urllib.pathname2url(
247 247 vcspath.join(wire['path'], path))
248 248 file_uri = svn.core.svn_path_canonicalize(abs_path)
249 249
250 250 start_rev = svn_opt_revision_value_t(0)
251 251 peg_rev = svn_opt_revision_value_t(revision)
252 252 end_rev = peg_rev
253 253
254 254 annotations = []
255 255
256 256 def receiver(line_no, revision, author, date, line, pool):
257 257 annotations.append((line_no, revision, line))
258 258
259 259 # TODO: Cannot use blame5, missing typemap function in the swig code
260 260 try:
261 261 svn.client.blame2(
262 262 file_uri, peg_rev, start_rev, end_rev,
263 263 receiver, svn.client.create_context())
264 264 except svn.core.SubversionException as exc:
265 265 log.exception("Error during blame operation.")
266 266 raise Exception(
267 267 "Blame not supported or file does not exist at path %s. "
268 268 "Error %s." % (path, exc))
269 269
270 270 return annotations
271 271
272 272 def get_node_type(self, wire, path, rev=None):
273 273 repo = self._factory.repo(wire)
274 274 fs_ptr = svn.repos.fs(repo)
275 275 if rev is None:
276 276 rev = svn.fs.youngest_rev(fs_ptr)
277 277 root = svn.fs.revision_root(fs_ptr, rev)
278 278 node = svn.fs.check_path(root, path)
279 279 return NODE_TYPE_MAPPING.get(node, None)
280 280
281 281 def get_nodes(self, wire, path, revision=None):
282 282 repo = self._factory.repo(wire)
283 283 fsobj = svn.repos.fs(repo)
284 284 if revision is None:
285 285 revision = svn.fs.youngest_rev(fsobj)
286 286 root = svn.fs.revision_root(fsobj, revision)
287 287 entries = svn.fs.dir_entries(root, path)
288 288 result = []
289 289 for entry_path, entry_info in entries.iteritems():
290 290 result.append(
291 291 (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
292 292 return result
293 293
294 294 def get_file_content(self, wire, path, rev=None):
295 295 repo = self._factory.repo(wire)
296 296 fsobj = svn.repos.fs(repo)
297 297 if rev is None:
298 298 rev = svn.fs.youngest_revision(fsobj)
299 299 root = svn.fs.revision_root(fsobj, rev)
300 300 content = svn.core.Stream(svn.fs.file_contents(root, path))
301 301 return content.read()
302 302
303 303 def get_file_size(self, wire, path, revision=None):
304 304 repo = self._factory.repo(wire)
305 305 fsobj = svn.repos.fs(repo)
306 306 if revision is None:
307 307 revision = svn.fs.youngest_revision(fsobj)
308 308 root = svn.fs.revision_root(fsobj, revision)
309 309 size = svn.fs.file_length(root, path)
310 310 return size
311 311
312 312 def create_repository(self, wire, compatible_version=None):
313 313 log.info('Creating Subversion repository in path "%s"', wire['path'])
314 314 self._factory.repo(wire, create=True,
315 315 compatible_version=compatible_version)
316 316
317 317 def import_remote_repository(self, wire, src_url):
318 318 repo_path = wire['path']
319 319 if not self.is_path_valid_repository(wire, repo_path):
320 320 raise Exception(
321 321 "Path %s is not a valid Subversion repository." % repo_path)
322 322 # TODO: johbo: URL checks ?
323 323 rdump = subprocess.Popen(
324 324 ['svnrdump', 'dump', '--non-interactive', src_url],
325 325 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
326 326 load = subprocess.Popen(
327 327 ['svnadmin', 'load', repo_path], stdin=rdump.stdout)
328 328
329 329 # TODO: johbo: This can be a very long operation, might be better
330 330 # to track some kind of status and provide an api to check if the
331 331 # import is done.
332 332 rdump.wait()
333 333 load.wait()
334 334
335 335 if rdump.returncode != 0:
336 336 errors = rdump.stderr.read()
337 337 log.error('svnrdump dump failed: statuscode %s: message: %s',
338 338 rdump.returncode, errors)
339 339 reason = 'UNKNOWN'
340 340 if 'svnrdump: E230001:' in errors:
341 341 reason = 'INVALID_CERTIFICATE'
342 342 raise Exception(
343 343 'Failed to dump the remote repository from %s.' % src_url,
344 344 reason)
345 345 if load.returncode != 0:
346 346 raise Exception(
347 347 'Failed to load the dump of remote repository from %s.' %
348 348 (src_url, ))
349 349
350 350 def commit(self, wire, message, author, timestamp, updated, removed):
351 351 assert isinstance(message, str)
352 352 assert isinstance(author, str)
353 353
354 354 repo = self._factory.repo(wire)
355 355 fsobj = svn.repos.fs(repo)
356 356
357 357 rev = svn.fs.youngest_rev(fsobj)
358 358 txn = svn.repos.fs_begin_txn_for_commit(repo, rev, author, message)
359 359 txn_root = svn.fs.txn_root(txn)
360 360
361 361 for node in updated:
362 362 TxnNodeProcessor(node, txn_root).update()
363 363 for node in removed:
364 364 TxnNodeProcessor(node, txn_root).remove()
365 365
366 366 commit_id = svn.repos.fs_commit_txn(repo, txn)
367 367
368 368 if timestamp:
369 369 apr_time = apr_time_t(timestamp)
370 370 ts_formatted = svn.core.svn_time_to_cstring(apr_time)
371 371 svn.fs.change_rev_prop(fsobj, commit_id, 'svn:date', ts_formatted)
372 372
373 373 log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
374 374 return commit_id
375 375
376 376 def diff(self, wire, rev1, rev2, path1=None, path2=None,
377 377 ignore_whitespace=False, context=3):
378
378 379 wire.update(cache=False)
379 380 repo = self._factory.repo(wire)
380 381 diff_creator = SvnDiffer(
381 382 repo, rev1, path1, rev2, path2, ignore_whitespace, context)
382 return diff_creator.generate_diff()
383 try:
384 return diff_creator.generate_diff()
385 except svn.core.SubversionException as e:
386 log.exception(
387 "Error during diff operation operation. "
388 "Path might not exist %s, %s" % (path1, path2))
389 return ""
383 390
384 391
385 392 class SvnDiffer(object):
386 393 """
387 394 Utility to create diffs based on difflib and the Subversion api
388 395 """
389 396
390 397 binary_content = False
391 398
392 399 def __init__(
393 400 self, repo, src_rev, src_path, tgt_rev, tgt_path,
394 401 ignore_whitespace, context):
395 402 self.repo = repo
396 403 self.ignore_whitespace = ignore_whitespace
397 404 self.context = context
398 405
399 406 fsobj = svn.repos.fs(repo)
400 407
401 408 self.tgt_rev = tgt_rev
402 409 self.tgt_path = tgt_path or ''
403 410 self.tgt_root = svn.fs.revision_root(fsobj, tgt_rev)
404 411 self.tgt_kind = svn.fs.check_path(self.tgt_root, self.tgt_path)
405 412
406 413 self.src_rev = src_rev
407 414 self.src_path = src_path or self.tgt_path
408 415 self.src_root = svn.fs.revision_root(fsobj, src_rev)
409 416 self.src_kind = svn.fs.check_path(self.src_root, self.src_path)
410 417
411 418 self._validate()
412 419
413 420 def _validate(self):
414 421 if (self.tgt_kind != svn.core.svn_node_none and
415 422 self.src_kind != svn.core.svn_node_none and
416 423 self.src_kind != self.tgt_kind):
417 424 # TODO: johbo: proper error handling
418 425 raise Exception(
419 426 "Source and target are not compatible for diff generation. "
420 427 "Source type: %s, target type: %s" %
421 428 (self.src_kind, self.tgt_kind))
422 429
423 430 def generate_diff(self):
424 431 buf = StringIO.StringIO()
425 432 if self.tgt_kind == svn.core.svn_node_dir:
426 433 self._generate_dir_diff(buf)
427 434 else:
428 435 self._generate_file_diff(buf)
429 436 return buf.getvalue()
430 437
431 438 def _generate_dir_diff(self, buf):
432 439 editor = DiffChangeEditor()
433 440 editor_ptr, editor_baton = svn.delta.make_editor(editor)
434 441 svn.repos.dir_delta2(
435 442 self.src_root,
436 443 self.src_path,
437 444 '', # src_entry
438 445 self.tgt_root,
439 446 self.tgt_path,
440 447 editor_ptr, editor_baton,
441 448 authorization_callback_allow_all,
442 449 False, # text_deltas
443 450 svn.core.svn_depth_infinity, # depth
444 451 False, # entry_props
445 452 False, # ignore_ancestry
446 453 )
447 454
448 455 for path, __, change in sorted(editor.changes):
449 456 self._generate_node_diff(
450 457 buf, change, path, self.tgt_path, path, self.src_path)
451 458
452 459 def _generate_file_diff(self, buf):
453 change = None
454 if self.src_kind == svn.core.svn_node_none:
455 change = "add"
456 elif self.tgt_kind == svn.core.svn_node_none:
457 change = "delete"
458 tgt_base, tgt_path = vcspath.split(self.tgt_path)
459 src_base, src_path = vcspath.split(self.src_path)
460 self._generate_node_diff(
461 buf, change, tgt_path, tgt_base, src_path, src_base)
460 change = None
461 if self.src_kind == svn.core.svn_node_none:
462 change = "add"
463 elif self.tgt_kind == svn.core.svn_node_none:
464 change = "delete"
465 tgt_base, tgt_path = vcspath.split(self.tgt_path)
466 src_base, src_path = vcspath.split(self.src_path)
467 self._generate_node_diff(
468 buf, change, tgt_path, tgt_base, src_path, src_base)
462 469
463 470 def _generate_node_diff(
464 471 self, buf, change, tgt_path, tgt_base, src_path, src_base):
472
473 if self.src_rev == self.tgt_rev and tgt_base == src_base:
474 # makes consistent behaviour with git/hg to return empty diff if
475 # we compare same revisions
476 return
477
465 478 tgt_full_path = vcspath.join(tgt_base, tgt_path)
466 479 src_full_path = vcspath.join(src_base, src_path)
467 480
468 481 self.binary_content = False
469 482 mime_type = self._get_mime_type(tgt_full_path)
483
470 484 if mime_type and not mime_type.startswith('text'):
471 485 self.binary_content = True
472 486 buf.write("=" * 67 + '\n')
473 487 buf.write("Cannot display: file marked as a binary type.\n")
474 488 buf.write("svn:mime-type = %s\n" % mime_type)
475 489 buf.write("Index: %s\n" % (tgt_path, ))
476 490 buf.write("=" * 67 + '\n')
477 491 buf.write("diff --git a/%(tgt_path)s b/%(tgt_path)s\n" % {
478 492 'tgt_path': tgt_path})
479 493
480 494 if change == 'add':
481 495 # TODO: johbo: SVN is missing a zero here compared to git
482 496 buf.write("new file mode 10644\n")
497
498 #TODO(marcink): intro to binary detection of svn patches
499 # if self.binary_content:
500 # buf.write('GIT binary patch\n')
501
483 502 buf.write("--- /dev/null\t(revision 0)\n")
484 503 src_lines = []
485 504 else:
486 505 if change == 'delete':
487 506 buf.write("deleted file mode 10644\n")
507
508 #TODO(marcink): intro to binary detection of svn patches
509 # if self.binary_content:
510 # buf.write('GIT binary patch\n')
511
488 512 buf.write("--- a/%s\t(revision %s)\n" % (
489 513 src_path, self.src_rev))
490 514 src_lines = self._svn_readlines(self.src_root, src_full_path)
491 515
492 516 if change == 'delete':
493 517 buf.write("+++ /dev/null\t(revision %s)\n" % (self.tgt_rev, ))
494 518 tgt_lines = []
495 519 else:
496 520 buf.write("+++ b/%s\t(revision %s)\n" % (
497 521 tgt_path, self.tgt_rev))
498 522 tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
499 523
500 524 if not self.binary_content:
501 525 udiff = svn_diff.unified_diff(
502 526 src_lines, tgt_lines, context=self.context,
503 527 ignore_blank_lines=self.ignore_whitespace,
504 528 ignore_case=False,
505 529 ignore_space_changes=self.ignore_whitespace)
506 530 buf.writelines(udiff)
507 531
508 532 def _get_mime_type(self, path):
509 533 try:
510 534 mime_type = svn.fs.node_prop(
511 535 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
512 536 except svn.core.SubversionException:
513 537 mime_type = svn.fs.node_prop(
514 538 self.src_root, path, svn.core.SVN_PROP_MIME_TYPE)
515 539 return mime_type
516 540
517 541 def _svn_readlines(self, fs_root, node_path):
518 542 if self.binary_content:
519 543 return []
520 544 node_kind = svn.fs.check_path(fs_root, node_path)
521 545 if node_kind not in (
522 546 svn.core.svn_node_file, svn.core.svn_node_symlink):
523 547 return []
524 548 content = svn.core.Stream(
525 549 svn.fs.file_contents(fs_root, node_path)).read()
526 550 return content.splitlines(True)
527 551
528 552
529 553 class DiffChangeEditor(svn.delta.Editor):
530 554 """
531 555 Records changes between two given revisions
532 556 """
533 557
534 558 def __init__(self):
535 559 self.changes = []
536 560
537 561 def delete_entry(self, path, revision, parent_baton, pool=None):
538 562 self.changes.append((path, None, 'delete'))
539 563
540 564 def add_file(
541 565 self, path, parent_baton, copyfrom_path, copyfrom_revision,
542 566 file_pool=None):
543 567 self.changes.append((path, 'file', 'add'))
544 568
545 569 def open_file(self, path, parent_baton, base_revision, file_pool=None):
546 570 self.changes.append((path, 'file', 'change'))
547 571
548 572
549 573 def authorization_callback_allow_all(root, path, pool):
550 574 return True
551 575
552 576
553 577 class TxnNodeProcessor(object):
554 578 """
555 579 Utility to process the change of one node within a transaction root.
556 580
557 581 It encapsulates the knowledge of how to add, update or remove
558 582 a node for a given transaction root. The purpose is to support the method
559 583 `SvnRemote.commit`.
560 584 """
561 585
562 586 def __init__(self, node, txn_root):
563 587 assert isinstance(node['path'], str)
564 588
565 589 self.node = node
566 590 self.txn_root = txn_root
567 591
568 592 def update(self):
569 593 self._ensure_parent_dirs()
570 594 self._add_file_if_node_does_not_exist()
571 595 self._update_file_content()
572 596 self._update_file_properties()
573 597
574 598 def remove(self):
575 599 svn.fs.delete(self.txn_root, self.node['path'])
576 600 # TODO: Clean up directory if empty
577 601
578 602 def _ensure_parent_dirs(self):
579 603 curdir = vcspath.dirname(self.node['path'])
580 604 dirs_to_create = []
581 605 while not self._svn_path_exists(curdir):
582 606 dirs_to_create.append(curdir)
583 607 curdir = vcspath.dirname(curdir)
584 608
585 609 for curdir in reversed(dirs_to_create):
586 610 log.debug('Creating missing directory "%s"', curdir)
587 611 svn.fs.make_dir(self.txn_root, curdir)
588 612
589 613 def _svn_path_exists(self, path):
590 614 path_status = svn.fs.check_path(self.txn_root, path)
591 615 return path_status != svn.core.svn_node_none
592 616
593 617 def _add_file_if_node_does_not_exist(self):
594 618 kind = svn.fs.check_path(self.txn_root, self.node['path'])
595 619 if kind == svn.core.svn_node_none:
596 620 svn.fs.make_file(self.txn_root, self.node['path'])
597 621
598 622 def _update_file_content(self):
599 623 assert isinstance(self.node['content'], str)
600 624 handler, baton = svn.fs.apply_textdelta(
601 625 self.txn_root, self.node['path'], None, None)
602 626 svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
603 627
604 628 def _update_file_properties(self):
605 629 properties = self.node.get('properties', {})
606 630 for key, value in properties.iteritems():
607 631 svn.fs.change_node_prop(
608 632 self.txn_root, self.node['path'], key, value)
609 633
610 634
611 635 def apr_time_t(timestamp):
612 636 """
613 637 Convert a Python timestamp into APR timestamp type apr_time_t
614 638 """
615 639 return timestamp * 1E6
616 640
617 641
618 642 def svn_opt_revision_value_t(num):
619 643 """
620 644 Put `num` into a `svn_opt_revision_value_t` structure.
621 645 """
622 646 value = svn.core.svn_opt_revision_value_t()
623 647 value.number = num
624 648 revision = svn.core.svn_opt_revision_t()
625 649 revision.kind = svn.core.svn_opt_revision_number
626 650 revision.value = value
627 651 return revision
@@ -1,207 +1,209 b''
1 1 # -*- coding: utf-8 -*-
2 2 #
3 3 # Copyright (C) 2004-2009 Edgewall Software
4 4 # Copyright (C) 2004-2006 Christopher Lenz <cmlenz@gmx.de>
5 5 # All rights reserved.
6 6 #
7 7 # This software is licensed as described in the file COPYING, which
8 8 # you should have received as part of this distribution. The terms
9 9 # are also available at http://trac.edgewall.org/wiki/TracLicense.
10 10 #
11 11 # This software consists of voluntary contributions made by many
12 12 # individuals. For the exact contribution history, see the revision
13 13 # history and logs, available at http://trac.edgewall.org/log/.
14 14 #
15 15 # Author: Christopher Lenz <cmlenz@gmx.de>
16 16
17 17 import difflib
18 18
19 19
20 20 def get_filtered_hunks(fromlines, tolines, context=None,
21 21 ignore_blank_lines=False, ignore_case=False,
22 22 ignore_space_changes=False):
23 23 """Retrieve differences in the form of `difflib.SequenceMatcher`
24 24 opcodes, grouped according to the ``context`` and ``ignore_*``
25 25 parameters.
26 26
27 27 :param fromlines: list of lines corresponding to the old content
28 28 :param tolines: list of lines corresponding to the new content
29 29 :param ignore_blank_lines: differences about empty lines only are ignored
30 30 :param ignore_case: upper case / lower case only differences are ignored
31 31 :param ignore_space_changes: differences in amount of spaces are ignored
32 32 :param context: the number of "equal" lines kept for representing
33 33 the context of the change
34 34 :return: generator of grouped `difflib.SequenceMatcher` opcodes
35 35
36 36 If none of the ``ignore_*`` parameters is `True`, there's nothing
37 37 to filter out the results will come straight from the
38 38 SequenceMatcher.
39 39 """
40 40 hunks = get_hunks(fromlines, tolines, context)
41 41 if ignore_space_changes or ignore_case or ignore_blank_lines:
42 42 hunks = filter_ignorable_lines(hunks, fromlines, tolines, context,
43 43 ignore_blank_lines, ignore_case,
44 44 ignore_space_changes)
45 45 return hunks
46 46
47 47
48 48 def get_hunks(fromlines, tolines, context=None):
49 49 """Generator yielding grouped opcodes describing differences .
50 50
51 51 See `get_filtered_hunks` for the parameter descriptions.
52 52 """
53 53 matcher = difflib.SequenceMatcher(None, fromlines, tolines)
54 54 if context is None:
55 55 return (hunk for hunk in [matcher.get_opcodes()])
56 56 else:
57 57 return matcher.get_grouped_opcodes(context)
58 58
59 59
60 60 def filter_ignorable_lines(hunks, fromlines, tolines, context,
61 61 ignore_blank_lines, ignore_case,
62 62 ignore_space_changes):
63 63 """Detect line changes that should be ignored and emits them as
64 64 tagged as "equal", possibly joined with the preceding and/or
65 65 following "equal" block.
66 66
67 67 See `get_filtered_hunks` for the parameter descriptions.
68 68 """
69 69 def is_ignorable(tag, fromlines, tolines):
70 70 if tag == 'delete' and ignore_blank_lines:
71 71 if ''.join(fromlines) == '':
72 72 return True
73 73 elif tag == 'insert' and ignore_blank_lines:
74 74 if ''.join(tolines) == '':
75 75 return True
76 76 elif tag == 'replace' and (ignore_case or ignore_space_changes):
77 77 if len(fromlines) != len(tolines):
78 78 return False
79 def f(str):
79
80 def f(input_str):
80 81 if ignore_case:
81 str = str.lower()
82 input_str = input_str.lower()
82 83 if ignore_space_changes:
83 str = ' '.join(str.split())
84 return str
84 input_str = ' '.join(input_str.split())
85 return input_str
86
85 87 for i in range(len(fromlines)):
86 88 if f(fromlines[i]) != f(tolines[i]):
87 89 return False
88 90 return True
89 91
90 92 hunks = list(hunks)
91 93 opcodes = []
92 94 ignored_lines = False
93 95 prev = None
94 96 for hunk in hunks:
95 97 for tag, i1, i2, j1, j2 in hunk:
96 98 if tag == 'equal':
97 99 if prev:
98 100 prev = (tag, prev[1], i2, prev[3], j2)
99 101 else:
100 102 prev = (tag, i1, i2, j1, j2)
101 103 else:
102 104 if is_ignorable(tag, fromlines[i1:i2], tolines[j1:j2]):
103 105 ignored_lines = True
104 106 if prev:
105 107 prev = 'equal', prev[1], i2, prev[3], j2
106 108 else:
107 109 prev = 'equal', i1, i2, j1, j2
108 110 continue
109 111 if prev:
110 112 opcodes.append(prev)
111 113 opcodes.append((tag, i1, i2, j1, j2))
112 114 prev = None
113 115 if prev:
114 116 opcodes.append(prev)
115 117
116 118 if ignored_lines:
117 119 if context is None:
118 120 yield opcodes
119 121 else:
120 122 # we leave at most n lines with the tag 'equal' before and after
121 123 # every change
122 124 n = context
123 125 nn = n + n
124 126
125 127 group = []
126 128 def all_equal():
127 129 all(op[0] == 'equal' for op in group)
128 130 for idx, (tag, i1, i2, j1, j2) in enumerate(opcodes):
129 131 if idx == 0 and tag == 'equal': # Fixup leading unchanged block
130 132 i1, j1 = max(i1, i2 - n), max(j1, j2 - n)
131 133 elif tag == 'equal' and i2 - i1 > nn:
132 134 group.append((tag, i1, min(i2, i1 + n), j1,
133 135 min(j2, j1 + n)))
134 136 if not all_equal():
135 137 yield group
136 138 group = []
137 139 i1, j1 = max(i1, i2 - n), max(j1, j2 - n)
138 140 group.append((tag, i1, i2, j1, j2))
139 141
140 142 if group and not (len(group) == 1 and group[0][0] == 'equal'):
141 143 if group[-1][0] == 'equal': # Fixup trailing unchanged block
142 144 tag, i1, i2, j1, j2 = group[-1]
143 145 group[-1] = tag, i1, min(i2, i1 + n), j1, min(j2, j1 + n)
144 146 if not all_equal():
145 147 yield group
146 148 else:
147 149 for hunk in hunks:
148 150 yield hunk
149 151
150 152
151 153 NO_NEWLINE_AT_END = '\\ No newline at end of file'
152 154
153 155
154 156 def unified_diff(fromlines, tolines, context=None, ignore_blank_lines=0,
155 157 ignore_case=0, ignore_space_changes=0, lineterm='\n'):
156 158 """
157 159 Generator producing lines corresponding to a textual diff.
158 160
159 161 See `get_filtered_hunks` for the parameter descriptions.
160 162 """
161 163 # TODO: johbo: Check if this can be nicely integrated into the matching
162 164 if ignore_space_changes:
163 165 fromlines = [l.strip() for l in fromlines]
164 166 tolines = [l.strip() for l in tolines]
165 167
166 168 for group in get_filtered_hunks(fromlines, tolines, context,
167 169 ignore_blank_lines, ignore_case,
168 170 ignore_space_changes):
169 171 i1, i2, j1, j2 = group[0][1], group[-1][2], group[0][3], group[-1][4]
170 172 if i1 == 0 and i2 == 0:
171 173 i1, i2 = -1, -1 # support for Add changes
172 174 if j1 == 0 and j2 == 0:
173 175 j1, j2 = -1, -1 # support for Delete changes
174 176 yield '@@ -%s +%s @@%s' % (
175 177 _hunk_range(i1 + 1, i2 - i1),
176 178 _hunk_range(j1 + 1, j2 - j1),
177 179 lineterm)
178 180 for tag, i1, i2, j1, j2 in group:
179 181 if tag == 'equal':
180 182 for line in fromlines[i1:i2]:
181 183 if not line.endswith(lineterm):
182 184 yield ' ' + line + lineterm
183 185 yield NO_NEWLINE_AT_END + lineterm
184 186 else:
185 187 yield ' ' + line
186 188 else:
187 189 if tag in ('replace', 'delete'):
188 190 for line in fromlines[i1:i2]:
189 191 if not line.endswith(lineterm):
190 192 yield '-' + line + lineterm
191 193 yield NO_NEWLINE_AT_END + lineterm
192 194 else:
193 195 yield '-' + line
194 196 if tag in ('replace', 'insert'):
195 197 for line in tolines[j1:j2]:
196 198 if not line.endswith(lineterm):
197 199 yield '+' + line + lineterm
198 200 yield NO_NEWLINE_AT_END + lineterm
199 201 else:
200 202 yield '+' + line
201 203
202 204
203 205 def _hunk_range(start, length):
204 206 if length != 1:
205 207 return '%d,%d' % (start, length)
206 208 else:
207 209 return '%d' % (start, )
@@ -1,57 +1,57 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import socket
19 19
20 20 import pytest
21 21
22 22
23 23 def pytest_addoption(parser):
24 24 parser.addoption(
25 25 '--repeat', type=int, default=100,
26 26 help="Number of repetitions in performance tests.")
27 27
28 28
29 29 @pytest.fixture(scope='session')
30 30 def repeat(request):
31 31 """
32 32 The number of repetitions is based on this fixture.
33 33
34 34 Slower calls may divide it by 10 or 100. It is chosen in a way so that the
35 35 tests are not too slow in our default test suite.
36 36 """
37 37 return request.config.getoption('--repeat')
38 38
39 39
40 40 @pytest.fixture(scope='session')
41 41 def vcsserver_port(request):
42 42 port = get_available_port()
43 43 print 'Using vcsserver port %s' % (port, )
44 44 return port
45 45
46 46
47 47 def get_available_port():
48 48 family = socket.AF_INET
49 49 socktype = socket.SOCK_STREAM
50 50 host = '127.0.0.1'
51 51
52 52 mysocket = socket.socket(family, socktype)
53 53 mysocket.bind((host, 0))
54 54 port = mysocket.getsockname()[1]
55 55 mysocket.close()
56 56 del mysocket
57 57 return port
@@ -1,71 +1,71 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import os
19 19 import shutil
20 20 import tempfile
21 21
22 22 import configobj
23 23
24 24
25 class TestINI(object):
25 class ContextINI(object):
26 26 """
27 27 Allows to create a new test.ini file as a copy of existing one with edited
28 28 data. If existing file is not present, it creates a new one. Example usage::
29 29
30 30 with TestINI('test.ini', [{'section': {'key': 'val'}}]) as new_test_ini_path:
31 31 print 'vcsserver --config=%s' % new_test_ini
32 32 """
33 33
34 34 def __init__(self, ini_file_path, ini_params, new_file_prefix=None,
35 35 destroy=True):
36 36 self.ini_file_path = ini_file_path
37 37 self.ini_params = ini_params
38 38 self.new_path = None
39 39 self.new_path_prefix = new_file_prefix or 'test'
40 40 self.destroy = destroy
41 41
42 42 def __enter__(self):
43 43 _, pref = tempfile.mkstemp()
44 44 loc = tempfile.gettempdir()
45 45 self.new_path = os.path.join(loc, '{}_{}_{}'.format(
46 46 pref, self.new_path_prefix, self.ini_file_path))
47 47
48 48 # copy ini file and modify according to the params, if we re-use a file
49 49 if os.path.isfile(self.ini_file_path):
50 50 shutil.copy(self.ini_file_path, self.new_path)
51 51 else:
52 52 # create new dump file for configObj to write to.
53 53 with open(self.new_path, 'wb'):
54 54 pass
55 55
56 56 config = configobj.ConfigObj(
57 57 self.new_path, file_error=True, write_empty_values=True)
58 58
59 59 for data in self.ini_params:
60 60 section, ini_params = data.items()[0]
61 61 key, val = ini_params.items()[0]
62 62 if section not in config:
63 63 config[section] = {}
64 64 config[section][key] = val
65 65
66 66 config.write()
67 67 return self.new_path
68 68
69 69 def __exit__(self, exc_type, exc_val, exc_tb):
70 70 if self.destroy:
71 71 os.remove(self.new_path)
@@ -1,162 +1,162 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import inspect
19 19
20 20 import pytest
21 21 import dulwich.errors
22 22 from mock import Mock, patch
23 23
24 24 from vcsserver import git
25 25
26 26
27 27 SAMPLE_REFS = {
28 28 'HEAD': 'fd627b9e0dd80b47be81af07c4a98518244ed2f7',
29 29 'refs/tags/v0.1.9': '341d28f0eec5ddf0b6b77871e13c2bbd6bec685c',
30 30 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
31 31 'refs/tags/v0.1.1': 'e6ea6d16e2f26250124a1f4b4fe37a912f9d86a0',
32 32 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
33 33 }
34 34
35 35
36 36 @pytest.fixture
37 37 def git_remote():
38 38 """
39 39 A GitRemote instance with a mock factory.
40 40 """
41 41 factory = Mock()
42 42 remote = git.GitRemote(factory)
43 43 return remote
44 44
45 45
46 46 def test_discover_git_version(git_remote):
47 47 version = git_remote.discover_git_version()
48 48 assert version
49 49
50 50
51 51 class TestGitFetch(object):
52 52 def setup(self):
53 53 self.mock_repo = Mock()
54 54 factory = Mock()
55 55 factory.repo = Mock(return_value=self.mock_repo)
56 56 self.remote_git = git.GitRemote(factory)
57 57
58 58 def test_fetches_all_when_no_commit_ids_specified(self):
59 59 def side_effect(determine_wants, *args, **kwargs):
60 60 determine_wants(SAMPLE_REFS)
61 61
62 62 with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch:
63 63 mock_fetch.side_effect = side_effect
64 64 self.remote_git.fetch(wire=None, url='/tmp/', apply_refs=False)
65 65 determine_wants = self.mock_repo.object_store.determine_wants_all
66 66 determine_wants.assert_called_once_with(SAMPLE_REFS)
67 67
68 68 def test_fetches_specified_commits(self):
69 69 selected_refs = {
70 70 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
71 71 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
72 72 }
73 73
74 74 def side_effect(determine_wants, *args, **kwargs):
75 75 result = determine_wants(SAMPLE_REFS)
76 76 assert sorted(result) == sorted(selected_refs.values())
77 77 return result
78 78
79 79 with patch('dulwich.client.LocalGitClient.fetch') as mock_fetch:
80 80 mock_fetch.side_effect = side_effect
81 81 self.remote_git.fetch(
82 82 wire=None, url='/tmp/', apply_refs=False,
83 83 refs=selected_refs.keys())
84 84 determine_wants = self.mock_repo.object_store.determine_wants_all
85 85 assert determine_wants.call_count == 0
86 86
87 87 def test_get_remote_refs(self):
88 88 factory = Mock()
89 89 remote_git = git.GitRemote(factory)
90 90 url = 'http://example.com/test/test.git'
91 91 sample_refs = {
92 92 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68',
93 93 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e',
94 94 }
95 95
96 96 with patch('vcsserver.git.Repo', create=False) as mock_repo:
97 97 mock_repo().get_refs.return_value = sample_refs
98 98 remote_refs = remote_git.get_remote_refs(wire=None, url=url)
99 99 mock_repo().get_refs.assert_called_once_with()
100 100 assert remote_refs == sample_refs
101 101
102 102 def test_remove_ref(self):
103 103 ref_to_remove = 'refs/tags/v0.1.9'
104 104 self.mock_repo.refs = SAMPLE_REFS.copy()
105 105 self.remote_git.remove_ref(None, ref_to_remove)
106 106 assert ref_to_remove not in self.mock_repo.refs
107 107
108 108
109 109 class TestReraiseSafeExceptions(object):
110 110 def test_method_decorated_with_reraise_safe_exceptions(self):
111 111 factory = Mock()
112 112 git_remote = git.GitRemote(factory)
113 113
114 114 def fake_function():
115 115 return None
116 116
117 117 decorator = git.reraise_safe_exceptions(fake_function)
118 118
119 119 methods = inspect.getmembers(git_remote, predicate=inspect.ismethod)
120 120 for method_name, method in methods:
121 121 if not method_name.startswith('_'):
122 122 assert method.im_func.__code__ == decorator.__code__
123 123
124 124 @pytest.mark.parametrize('side_effect, expected_type', [
125 125 (dulwich.errors.ChecksumMismatch('0000000', 'deadbeef'), 'lookup'),
126 126 (dulwich.errors.NotCommitError('deadbeef'), 'lookup'),
127 127 (dulwich.errors.MissingCommitError('deadbeef'), 'lookup'),
128 128 (dulwich.errors.ObjectMissing('deadbeef'), 'lookup'),
129 129 (dulwich.errors.HangupException(), 'error'),
130 130 (dulwich.errors.UnexpectedCommandError('test-cmd'), 'error'),
131 131 ])
132 132 def test_safe_exceptions_reraised(self, side_effect, expected_type):
133 133 @git.reraise_safe_exceptions
134 134 def fake_method():
135 135 raise side_effect
136 136
137 137 with pytest.raises(Exception) as exc_info:
138 138 fake_method()
139 139 assert type(exc_info.value) == Exception
140 140 assert exc_info.value._vcs_kind == expected_type
141 141
142 142
143 143 class TestDulwichRepoWrapper(object):
144 144 def test_calls_close_on_delete(self):
145 145 isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True)
146 146 with isdir_patcher:
147 147 repo = git.Repo('/tmp/abcde')
148 148 with patch.object(git.DulwichRepo, 'close') as close_mock:
149 149 del repo
150 150 close_mock.assert_called_once_with()
151 151
152 152
153 153 class TestGitFactory(object):
154 154 def test_create_repo_returns_dulwich_wrapper(self):
155 155 factory = git.GitFactory(repo_cache=Mock())
156 156 wire = {
157 157 'path': '/tmp/abcde'
158 158 }
159 159 isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True)
160 160 with isdir_patcher:
161 161 result = factory._create_repo(wire, True)
162 162 assert isinstance(result, git.Repo)
@@ -1,127 +1,127 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import inspect
19 19 import sys
20 20 import traceback
21 21
22 22 import pytest
23 23 from mercurial.error import LookupError
24 24 from mock import Mock, MagicMock, patch
25 25
26 26 from vcsserver import exceptions, hg, hgcompat
27 27
28 28
29 29 class TestHGLookup(object):
30 30 def setup(self):
31 31 self.mock_repo = MagicMock()
32 32 self.mock_repo.__getitem__.side_effect = LookupError(
33 33 'revision_or_commit_id', 'index', 'message')
34 34 factory = Mock()
35 35 factory.repo = Mock(return_value=self.mock_repo)
36 36 self.remote_hg = hg.HgRemote(factory)
37 37
38 38 def test_fail_lookup_hg(self):
39 39 with pytest.raises(Exception) as exc_info:
40 40 self.remote_hg.lookup(
41 41 wire=None, revision='revision_or_commit_id', both=True)
42 42
43 43 assert exc_info.value._vcs_kind == 'lookup'
44 44 assert 'revision_or_commit_id' in exc_info.value.args
45 45
46 46
47 47 class TestDiff(object):
48 48 def test_raising_safe_exception_when_lookup_failed(self):
49 49 repo = Mock()
50 50 factory = Mock()
51 51 factory.repo = Mock(return_value=repo)
52 52 hg_remote = hg.HgRemote(factory)
53 53 with patch('mercurial.patch.diff') as diff_mock:
54 54 diff_mock.side_effect = LookupError(
55 55 'deadbeef', 'index', 'message')
56 56 with pytest.raises(Exception) as exc_info:
57 57 hg_remote.diff(
58 58 wire=None, rev1='deadbeef', rev2='deadbee1',
59 59 file_filter=None, opt_git=True, opt_ignorews=True,
60 60 context=3)
61 61 assert type(exc_info.value) == Exception
62 62 assert exc_info.value._vcs_kind == 'lookup'
63 63
64 64
65 65 class TestReraiseSafeExceptions(object):
66 66 def test_method_decorated_with_reraise_safe_exceptions(self):
67 67 factory = Mock()
68 68 hg_remote = hg.HgRemote(factory)
69 69 methods = inspect.getmembers(hg_remote, predicate=inspect.ismethod)
70 70 decorator = hg.reraise_safe_exceptions(None)
71 71 for method_name, method in methods:
72 72 if not method_name.startswith('_'):
73 73 assert method.im_func.__code__ == decorator.__code__
74 74
75 75 @pytest.mark.parametrize('side_effect, expected_type', [
76 76 (hgcompat.Abort(), 'abort'),
77 77 (hgcompat.InterventionRequired(), 'abort'),
78 78 (hgcompat.RepoLookupError(), 'lookup'),
79 79 (hgcompat.LookupError('deadbeef', 'index', 'message'), 'lookup'),
80 80 (hgcompat.RepoError(), 'error'),
81 81 (hgcompat.RequirementError(), 'requirement'),
82 82 ])
83 83 def test_safe_exceptions_reraised(self, side_effect, expected_type):
84 84 @hg.reraise_safe_exceptions
85 85 def fake_method():
86 86 raise side_effect
87 87
88 88 with pytest.raises(Exception) as exc_info:
89 89 fake_method()
90 90 assert type(exc_info.value) == Exception
91 91 assert exc_info.value._vcs_kind == expected_type
92 92
93 93 def test_keeps_original_traceback(self):
94 94 @hg.reraise_safe_exceptions
95 95 def fake_method():
96 96 try:
97 97 raise hgcompat.Abort()
98 98 except:
99 99 self.original_traceback = traceback.format_tb(
100 100 sys.exc_info()[2])
101 101 raise
102 102
103 103 try:
104 104 fake_method()
105 105 except Exception:
106 106 new_traceback = traceback.format_tb(sys.exc_info()[2])
107 107
108 108 new_traceback_tail = new_traceback[-len(self.original_traceback):]
109 109 assert new_traceback_tail == self.original_traceback
110 110
111 111 def test_maps_unknow_exceptions_to_unhandled(self):
112 112 @hg.reraise_safe_exceptions
113 113 def stub_method():
114 114 raise ValueError('stub')
115 115
116 116 with pytest.raises(Exception) as exc_info:
117 117 stub_method()
118 118 assert exc_info.value._vcs_kind == 'unhandled'
119 119
120 120 def test_does_not_map_known_exceptions(self):
121 121 @hg.reraise_safe_exceptions
122 122 def stub_method():
123 123 raise exceptions.LookupException('stub')
124 124
125 125 with pytest.raises(Exception) as exc_info:
126 126 stub_method()
127 127 assert exc_info.value._vcs_kind == 'lookup'
@@ -1,125 +1,125 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import mock
19 19 import pytest
20 20
21 21 from vcsserver import hgcompat, hgpatches
22 22
23 23
24 24 LARGEFILES_CAPABILITY = 'largefiles=serve'
25 25
26 26
27 27 def test_patch_largefiles_capabilities_applies_patch(
28 28 patched_capabilities):
29 29 lfproto = hgcompat.largefiles.proto
30 30 hgpatches.patch_largefiles_capabilities()
31 31 assert lfproto.capabilities.func_name == '_dynamic_capabilities'
32 32
33 33
34 34 def test_dynamic_capabilities_uses_original_function_if_not_enabled(
35 35 stub_repo, stub_proto, stub_ui, stub_extensions, patched_capabilities):
36 36 dynamic_capabilities = hgpatches._dynamic_capabilities_wrapper(
37 37 hgcompat.largefiles.proto, stub_extensions)
38 38
39 39 caps = dynamic_capabilities(stub_repo, stub_proto)
40 40
41 41 stub_extensions.assert_called_once_with(stub_ui)
42 42 assert LARGEFILES_CAPABILITY not in caps
43 43
44 44
45 45 def test_dynamic_capabilities_uses_updated_capabilitiesorig(
46 46 stub_repo, stub_proto, stub_ui, stub_extensions, patched_capabilities):
47 47 dynamic_capabilities = hgpatches._dynamic_capabilities_wrapper(
48 48 hgcompat.largefiles.proto, stub_extensions)
49 49
50 50 # This happens when the extension is loaded for the first time, important
51 51 # to ensure that an updated function is correctly picked up.
52 52 hgcompat.largefiles.proto.capabilitiesorig = mock.Mock(
53 53 return_value='REPLACED')
54 54
55 55 caps = dynamic_capabilities(stub_repo, stub_proto)
56 56 assert 'REPLACED' == caps
57 57
58 58
59 59 def test_dynamic_capabilities_ignores_updated_capabilities(
60 60 stub_repo, stub_proto, stub_ui, stub_extensions, patched_capabilities):
61 61 stub_extensions.return_value = [('largefiles', mock.Mock())]
62 62 dynamic_capabilities = hgpatches._dynamic_capabilities_wrapper(
63 63 hgcompat.largefiles.proto, stub_extensions)
64 64
65 65 # This happens when the extension is loaded for the first time, important
66 66 # to ensure that an updated function is correctly picked up.
67 67 hgcompat.largefiles.proto.capabilities = mock.Mock(
68 68 side_effect=Exception('Must not be called'))
69 69
70 70 dynamic_capabilities(stub_repo, stub_proto)
71 71
72 72
73 73 def test_dynamic_capabilities_uses_largefiles_if_enabled(
74 74 stub_repo, stub_proto, stub_ui, stub_extensions, patched_capabilities):
75 75 stub_extensions.return_value = [('largefiles', mock.Mock())]
76 76
77 77 dynamic_capabilities = hgpatches._dynamic_capabilities_wrapper(
78 78 hgcompat.largefiles.proto, stub_extensions)
79 79
80 80 caps = dynamic_capabilities(stub_repo, stub_proto)
81 81
82 82 stub_extensions.assert_called_once_with(stub_ui)
83 83 assert LARGEFILES_CAPABILITY in caps
84 84
85 85
86 86 @pytest.fixture
87 87 def patched_capabilities(request):
88 88 """
89 89 Patch in `capabilitiesorig` and restore both capability functions.
90 90 """
91 91 lfproto = hgcompat.largefiles.proto
92 92 orig_capabilities = lfproto.capabilities
93 93 orig_capabilitiesorig = lfproto.capabilitiesorig
94 94
95 95 lfproto.capabilitiesorig = mock.Mock(return_value='ORIG')
96 96
97 97 @request.addfinalizer
98 98 def restore():
99 99 lfproto.capabilities = orig_capabilities
100 100 lfproto.capabilitiesorig = orig_capabilitiesorig
101 101
102 102
103 103 @pytest.fixture
104 104 def stub_repo(stub_ui):
105 105 repo = mock.Mock()
106 106 repo.ui = stub_ui
107 107 return repo
108 108
109 109
110 110 @pytest.fixture
111 111 def stub_proto(stub_ui):
112 112 proto = mock.Mock()
113 113 proto.ui = stub_ui
114 114 return proto
115 115
116 116
117 117 @pytest.fixture
118 118 def stub_ui():
119 119 return hgcompat.ui.ui()
120 120
121 121
122 122 @pytest.fixture
123 123 def stub_extensions():
124 124 extensions = mock.Mock(return_value=tuple())
125 125 return extensions
@@ -1,549 +1,549 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import contextlib
19 19 import io
20 20 import threading
21 21 from BaseHTTPServer import BaseHTTPRequestHandler
22 22 from SocketServer import TCPServer
23 23
24 24 import mercurial.ui
25 25 import mock
26 26 import pytest
27 27 import simplejson as json
28 28
29 29 from vcsserver import hooks
30 30
31 31
32 32 class HooksStub(object):
33 33 """
34 34 Simulates a Proy4.Proxy object.
35 35
36 36 Will always return `result`, no matter which hook has been called on it.
37 37 """
38 38
39 39 def __init__(self, result):
40 40 self._result = result
41 41
42 42 def __call__(self, hooks_uri):
43 43 return self
44 44
45 45 def __enter__(self):
46 46 return self
47 47
48 48 def __exit__(self, exc_type, exc_value, traceback):
49 49 pass
50 50
51 51 def __getattr__(self, name):
52 52 return mock.Mock(return_value=self._result)
53 53
54 54
55 55 @contextlib.contextmanager
56 56 def mock_hook_response(
57 57 status=0, output='', exception=None, exception_args=None):
58 58 response = {
59 59 'status': status,
60 60 'output': output,
61 61 }
62 62 if exception:
63 63 response.update({
64 64 'exception': exception,
65 65 'exception_args': exception_args,
66 66 })
67 67
68 68 with mock.patch('Pyro4.Proxy', HooksStub(response)):
69 69 yield
70 70
71 71
72 72 def get_hg_ui(extras=None):
73 73 """Create a Config object with a valid RC_SCM_DATA entry."""
74 74 extras = extras or {}
75 75 required_extras = {
76 76 'username': '',
77 77 'repository': '',
78 78 'locked_by': '',
79 79 'scm': '',
80 80 'make_lock': '',
81 81 'action': '',
82 82 'ip': '',
83 83 'hooks_uri': 'fake_hooks_uri',
84 84 }
85 85 required_extras.update(extras)
86 86 hg_ui = mercurial.ui.ui()
87 87 hg_ui.setconfig('rhodecode', 'RC_SCM_DATA', json.dumps(required_extras))
88 88
89 89 return hg_ui
90 90
91 91
92 92 def test_call_hook_no_error(capsys):
93 93 extras = {
94 94 'hooks_uri': 'fake_hook_uri',
95 95 }
96 96 expected_output = 'My mock outptut'
97 97 writer = mock.Mock()
98 98
99 99 with mock_hook_response(status=1, output=expected_output):
100 100 hooks._call_hook('hook_name', extras, writer)
101 101
102 102 out, err = capsys.readouterr()
103 103
104 104 writer.write.assert_called_with(expected_output)
105 105 assert err == ''
106 106
107 107
108 108 def test_call_hook_with_exception(capsys):
109 109 extras = {
110 110 'hooks_uri': 'fake_hook_uri',
111 111 }
112 112 expected_output = 'My mock outptut'
113 113 writer = mock.Mock()
114 114
115 115 with mock_hook_response(status=1, output=expected_output,
116 116 exception='TypeError',
117 117 exception_args=('Mock exception', )):
118 118 with pytest.raises(Exception) as excinfo:
119 119 hooks._call_hook('hook_name', extras, writer)
120 120
121 121 assert excinfo.type == Exception
122 122 assert 'Mock exception' in str(excinfo.value)
123 123
124 124 out, err = capsys.readouterr()
125 125
126 126 writer.write.assert_called_with(expected_output)
127 127 assert err == ''
128 128
129 129
130 130 def test_call_hook_with_locked_exception(capsys):
131 131 extras = {
132 132 'hooks_uri': 'fake_hook_uri',
133 133 }
134 134 expected_output = 'My mock outptut'
135 135 writer = mock.Mock()
136 136
137 137 with mock_hook_response(status=1, output=expected_output,
138 138 exception='HTTPLockedRC',
139 139 exception_args=('message',)):
140 140 with pytest.raises(Exception) as excinfo:
141 141 hooks._call_hook('hook_name', extras, writer)
142 142
143 143 assert excinfo.value._vcs_kind == 'repo_locked'
144 144 assert 'message' == str(excinfo.value)
145 145
146 146 out, err = capsys.readouterr()
147 147
148 148 writer.write.assert_called_with(expected_output)
149 149 assert err == ''
150 150
151 151
152 152 def test_call_hook_with_stdout():
153 153 extras = {
154 154 'hooks_uri': 'fake_hook_uri',
155 155 }
156 156 expected_output = 'My mock outptut'
157 157
158 158 stdout = io.BytesIO()
159 159 with mock_hook_response(status=1, output=expected_output):
160 160 hooks._call_hook('hook_name', extras, stdout)
161 161
162 162 assert stdout.getvalue() == expected_output
163 163
164 164
165 165 def test_repo_size():
166 166 hg_ui = get_hg_ui()
167 167
168 168 with mock_hook_response(status=1):
169 169 assert hooks.repo_size(hg_ui, None) == 1
170 170
171 171
172 172 def test_pre_pull():
173 173 hg_ui = get_hg_ui()
174 174
175 175 with mock_hook_response(status=1):
176 176 assert hooks.pre_pull(hg_ui, None) == 1
177 177
178 178
179 179 def test_post_pull():
180 180 hg_ui = get_hg_ui()
181 181
182 182 with mock_hook_response(status=1):
183 183 assert hooks.post_pull(hg_ui, None) == 1
184 184
185 185
186 186 def test_pre_push():
187 187 hg_ui = get_hg_ui()
188 188
189 189 with mock_hook_response(status=1):
190 190 assert hooks.pre_push(hg_ui, None) == 1
191 191
192 192
193 193 def test_post_push():
194 194 hg_ui = get_hg_ui()
195 195
196 196 with mock_hook_response(status=1):
197 197 with mock.patch('vcsserver.hooks._rev_range_hash', return_value=[]):
198 198 assert hooks.post_push(hg_ui, None, None) == 1
199 199
200 200
201 201 def test_git_pre_receive():
202 202 extras = {
203 203 'hooks': ['push'],
204 204 'hooks_uri': 'fake_hook_uri',
205 205 }
206 206 with mock_hook_response(status=1):
207 207 response = hooks.git_pre_receive(None, None,
208 208 {'RC_SCM_DATA': json.dumps(extras)})
209 209 assert response == 1
210 210
211 211
212 212 def test_git_pre_receive_is_disabled():
213 213 extras = {'hooks': ['pull']}
214 214 response = hooks.git_pre_receive(None, None,
215 215 {'RC_SCM_DATA': json.dumps(extras)})
216 216
217 217 assert response == 0
218 218
219 219
220 220 def test_git_post_receive_no_subprocess_call():
221 221 extras = {
222 222 'hooks': ['push'],
223 223 'hooks_uri': 'fake_hook_uri',
224 224 }
225 225 # Setting revision_lines to '' avoid all subprocess_calls
226 226 with mock_hook_response(status=1):
227 227 response = hooks.git_post_receive(None, '',
228 228 {'RC_SCM_DATA': json.dumps(extras)})
229 229 assert response == 1
230 230
231 231
232 232 def test_git_post_receive_is_disabled():
233 233 extras = {'hooks': ['pull']}
234 234 response = hooks.git_post_receive(None, '',
235 235 {'RC_SCM_DATA': json.dumps(extras)})
236 236
237 237 assert response == 0
238 238
239 239
240 240 def test_git_post_receive_calls_repo_size():
241 241 extras = {'hooks': ['push', 'repo_size']}
242 242 with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
243 243 hooks.git_post_receive(
244 244 None, '', {'RC_SCM_DATA': json.dumps(extras)})
245 245 extras.update({'commit_ids': []})
246 246 expected_calls = [
247 247 mock.call('repo_size', extras, mock.ANY),
248 248 mock.call('post_push', extras, mock.ANY),
249 249 ]
250 250 assert call_hook_mock.call_args_list == expected_calls
251 251
252 252
253 253 def test_git_post_receive_does_not_call_disabled_repo_size():
254 254 extras = {'hooks': ['push']}
255 255 with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
256 256 hooks.git_post_receive(
257 257 None, '', {'RC_SCM_DATA': json.dumps(extras)})
258 258 extras.update({'commit_ids': []})
259 259 expected_calls = [
260 260 mock.call('post_push', extras, mock.ANY)
261 261 ]
262 262 assert call_hook_mock.call_args_list == expected_calls
263 263
264 264
265 265 def test_repo_size_exception_does_not_affect_git_post_receive():
266 266 extras = {'hooks': ['push', 'repo_size']}
267 267 status = 0
268 268
269 269 def side_effect(name, *args, **kwargs):
270 270 if name == 'repo_size':
271 271 raise Exception('Fake exception')
272 272 else:
273 273 return status
274 274
275 275 with mock.patch.object(hooks, '_call_hook') as call_hook_mock:
276 276 call_hook_mock.side_effect = side_effect
277 277 result = hooks.git_post_receive(
278 278 None, '', {'RC_SCM_DATA': json.dumps(extras)})
279 279 assert result == status
280 280
281 281
282 282 @mock.patch('vcsserver.hooks._run_command')
283 283 def test_git_post_receive_first_commit_sub_branch(cmd_mock):
284 284 def cmd_mock_returns(args):
285 285 if args == ['git', 'show', 'HEAD']:
286 286 raise
287 287 if args == ['git', 'for-each-ref', '--format=%(refname)',
288 288 'refs/heads/*']:
289 289 return 'refs/heads/test-branch2/sub-branch'
290 290 if args == ['git', 'log', '--reverse', '--pretty=format:%H', '--',
291 291 '9695eef57205c17566a3ae543be187759b310bb7', '--not',
292 292 'refs/heads/test-branch2/sub-branch']:
293 293 return ''
294 294
295 295 cmd_mock.side_effect = cmd_mock_returns
296 296
297 297 extras = {
298 298 'hooks': ['push'],
299 299 'hooks_uri': 'fake_hook_uri'
300 300 }
301 301 rev_lines = ['0000000000000000000000000000000000000000 '
302 302 '9695eef57205c17566a3ae543be187759b310bb7 '
303 303 'refs/heads/feature/sub-branch\n']
304 304 with mock_hook_response(status=0):
305 305 response = hooks.git_post_receive(None, rev_lines,
306 306 {'RC_SCM_DATA': json.dumps(extras)})
307 307
308 308 calls = [
309 309 mock.call(['git', 'show', 'HEAD']),
310 310 mock.call(['git', 'symbolic-ref', 'HEAD',
311 311 'refs/heads/feature/sub-branch']),
312 312 ]
313 313 cmd_mock.assert_has_calls(calls, any_order=True)
314 314 assert response == 0
315 315
316 316
317 317 @mock.patch('vcsserver.hooks._run_command')
318 318 def test_git_post_receive_first_commit_revs(cmd_mock):
319 319 extras = {
320 320 'hooks': ['push'],
321 321 'hooks_uri': 'fake_hook_uri'
322 322 }
323 323 rev_lines = [
324 324 '0000000000000000000000000000000000000000 '
325 325 '9695eef57205c17566a3ae543be187759b310bb7 refs/heads/master\n']
326 326 with mock_hook_response(status=0):
327 327 response = hooks.git_post_receive(
328 328 None, rev_lines, {'RC_SCM_DATA': json.dumps(extras)})
329 329
330 330 calls = [
331 331 mock.call(['git', 'show', 'HEAD']),
332 332 mock.call(['git', 'for-each-ref', '--format=%(refname)',
333 333 'refs/heads/*']),
334 334 mock.call(['git', 'log', '--reverse', '--pretty=format:%H',
335 335 '--', '9695eef57205c17566a3ae543be187759b310bb7', '--not',
336 336 ''])
337 337 ]
338 338 cmd_mock.assert_has_calls(calls, any_order=True)
339 339
340 340 assert response == 0
341 341
342 342
343 343 def test_git_pre_pull():
344 344 extras = {
345 345 'hooks': ['pull'],
346 346 'hooks_uri': 'fake_hook_uri',
347 347 }
348 348 with mock_hook_response(status=1, output='foo'):
349 349 assert hooks.git_pre_pull(extras) == hooks.HookResponse(1, 'foo')
350 350
351 351
352 352 def test_git_pre_pull_exception_is_caught():
353 353 extras = {
354 354 'hooks': ['pull'],
355 355 'hooks_uri': 'fake_hook_uri',
356 356 }
357 357 with mock_hook_response(status=2, exception=Exception('foo')):
358 358 assert hooks.git_pre_pull(extras).status == 128
359 359
360 360
361 361 def test_git_pre_pull_is_disabled():
362 362 assert hooks.git_pre_pull({'hooks': ['push']}) == hooks.HookResponse(0, '')
363 363
364 364
365 365 def test_git_post_pull():
366 366 extras = {
367 367 'hooks': ['pull'],
368 368 'hooks_uri': 'fake_hook_uri',
369 369 }
370 370 with mock_hook_response(status=1, output='foo'):
371 371 assert hooks.git_post_pull(extras) == hooks.HookResponse(1, 'foo')
372 372
373 373
374 374 def test_git_post_pull_exception_is_caught():
375 375 extras = {
376 376 'hooks': ['pull'],
377 377 'hooks_uri': 'fake_hook_uri',
378 378 }
379 379 with mock_hook_response(status=2, exception='Exception',
380 380 exception_args=('foo',)):
381 381 assert hooks.git_post_pull(extras).status == 128
382 382
383 383
384 384 def test_git_post_pull_is_disabled():
385 385 assert (
386 386 hooks.git_post_pull({'hooks': ['push']}) == hooks.HookResponse(0, ''))
387 387
388 388
389 389 class TestGetHooksClient(object):
390 390 def test_returns_pyro_client_when_protocol_matches(self):
391 391 hooks_uri = 'localhost:8000'
392 392 result = hooks._get_hooks_client({
393 393 'hooks_uri': hooks_uri,
394 394 'hooks_protocol': 'pyro4'
395 395 })
396 396 assert isinstance(result, hooks.HooksPyro4Client)
397 397 assert result.hooks_uri == hooks_uri
398 398
399 399 def test_returns_http_client_when_protocol_matches(self):
400 400 hooks_uri = 'localhost:8000'
401 401 result = hooks._get_hooks_client({
402 402 'hooks_uri': hooks_uri,
403 403 'hooks_protocol': 'http'
404 404 })
405 405 assert isinstance(result, hooks.HooksHttpClient)
406 406 assert result.hooks_uri == hooks_uri
407 407
408 408 def test_returns_pyro4_client_when_no_protocol_is_specified(self):
409 409 hooks_uri = 'localhost:8000'
410 410 result = hooks._get_hooks_client({
411 411 'hooks_uri': hooks_uri
412 412 })
413 413 assert isinstance(result, hooks.HooksPyro4Client)
414 414 assert result.hooks_uri == hooks_uri
415 415
416 416 def test_returns_dummy_client_when_hooks_uri_not_specified(self):
417 417 fake_module = mock.Mock()
418 418 import_patcher = mock.patch.object(
419 419 hooks.importlib, 'import_module', return_value=fake_module)
420 420 fake_module_name = 'fake.module'
421 421 with import_patcher as import_mock:
422 422 result = hooks._get_hooks_client(
423 423 {'hooks_module': fake_module_name})
424 424
425 425 import_mock.assert_called_once_with(fake_module_name)
426 426 assert isinstance(result, hooks.HooksDummyClient)
427 427 assert result._hooks_module == fake_module
428 428
429 429
430 430 class TestHooksHttpClient(object):
431 431 def test_init_sets_hooks_uri(self):
432 432 uri = 'localhost:3000'
433 433 client = hooks.HooksHttpClient(uri)
434 434 assert client.hooks_uri == uri
435 435
436 436 def test_serialize_returns_json_string(self):
437 437 client = hooks.HooksHttpClient('localhost:3000')
438 438 hook_name = 'test'
439 439 extras = {
440 440 'first': 1,
441 441 'second': 'two'
442 442 }
443 443 result = client._serialize(hook_name, extras)
444 444 expected_result = json.dumps({
445 445 'method': hook_name,
446 446 'extras': extras
447 447 })
448 448 assert result == expected_result
449 449
450 450 def test_call_queries_http_server(self, http_mirror):
451 451 client = hooks.HooksHttpClient(http_mirror.uri)
452 452 hook_name = 'test'
453 453 extras = {
454 454 'first': 1,
455 455 'second': 'two'
456 456 }
457 457 result = client(hook_name, extras)
458 458 expected_result = {
459 459 'method': hook_name,
460 460 'extras': extras
461 461 }
462 462 assert result == expected_result
463 463
464 464
465 465 class TestHooksDummyClient(object):
466 466 def test_init_imports_hooks_module(self):
467 467 hooks_module_name = 'rhodecode.fake.module'
468 468 hooks_module = mock.MagicMock()
469 469
470 470 import_patcher = mock.patch.object(
471 471 hooks.importlib, 'import_module', return_value=hooks_module)
472 472 with import_patcher as import_mock:
473 473 client = hooks.HooksDummyClient(hooks_module_name)
474 474 import_mock.assert_called_once_with(hooks_module_name)
475 475 assert client._hooks_module == hooks_module
476 476
477 477 def test_call_returns_hook_result(self):
478 478 hooks_module_name = 'rhodecode.fake.module'
479 479 hooks_module = mock.MagicMock()
480 480 import_patcher = mock.patch.object(
481 481 hooks.importlib, 'import_module', return_value=hooks_module)
482 482 with import_patcher:
483 483 client = hooks.HooksDummyClient(hooks_module_name)
484 484
485 485 result = client('post_push', {})
486 486 hooks_module.Hooks.assert_called_once_with()
487 487 assert result == hooks_module.Hooks().__enter__().post_push()
488 488
489 489
490 490 class TestHooksPyro4Client(object):
491 491 def test_init_sets_hooks_uri(self):
492 492 uri = 'localhost:3000'
493 493 client = hooks.HooksPyro4Client(uri)
494 494 assert client.hooks_uri == uri
495 495
496 496 def test_call_returns_hook_value(self):
497 497 hooks_uri = 'localhost:3000'
498 498 client = hooks.HooksPyro4Client(hooks_uri)
499 499 hooks_module = mock.Mock()
500 500 context_manager = mock.MagicMock()
501 501 context_manager.__enter__.return_value = hooks_module
502 502 pyro4_patcher = mock.patch.object(
503 503 hooks.Pyro4, 'Proxy', return_value=context_manager)
504 504 extras = {
505 505 'test': 'test'
506 506 }
507 507 with pyro4_patcher as pyro4_mock:
508 508 result = client('post_push', extras)
509 509 pyro4_mock.assert_called_once_with(hooks_uri)
510 510 hooks_module.post_push.assert_called_once_with(extras)
511 511 assert result == hooks_module.post_push.return_value
512 512
513 513
514 514 @pytest.fixture
515 515 def http_mirror(request):
516 516 server = MirrorHttpServer()
517 517 request.addfinalizer(server.stop)
518 518 return server
519 519
520 520
521 521 class MirrorHttpHandler(BaseHTTPRequestHandler):
522 522 def do_POST(self):
523 523 length = int(self.headers['Content-Length'])
524 524 body = self.rfile.read(length).decode('utf-8')
525 525 self.send_response(200)
526 526 self.end_headers()
527 527 self.wfile.write(body)
528 528
529 529
530 530 class MirrorHttpServer(object):
531 531 ip_address = '127.0.0.1'
532 532 port = 0
533 533
534 534 def __init__(self):
535 535 self._daemon = TCPServer((self.ip_address, 0), MirrorHttpHandler)
536 536 _, self.port = self._daemon.server_address
537 537 self._thread = threading.Thread(target=self._daemon.serve_forever)
538 538 self._thread.daemon = True
539 539 self._thread.start()
540 540
541 541 def stop(self):
542 542 self._daemon.shutdown()
543 543 self._thread.join()
544 544 self._daemon = None
545 545 self._thread = None
546 546
547 547 @property
548 548 def uri(self):
549 549 return '{}:{}'.format(self.ip_address, self.port)
1 NO CONTENT: file renamed from tests/test_http_performance.py to vcsserver/tests/test_http_performance.py
@@ -1,57 +1,57 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import mock
19 19 import pytest
20 20
21 21 from vcsserver import main
22 22 from vcsserver.base import obfuscate_qs
23 23
24 24
25 25 @mock.patch('vcsserver.main.VcsServerCommand', mock.Mock())
26 26 @mock.patch('vcsserver.hgpatches.patch_largefiles_capabilities')
27 27 def test_applies_largefiles_patch(patch_largefiles_capabilities):
28 28 main.main([])
29 29 patch_largefiles_capabilities.assert_called_once_with()
30 30
31 31
32 32 @mock.patch('vcsserver.main.VcsServerCommand', mock.Mock())
33 33 @mock.patch('vcsserver.main.MercurialFactory', None)
34 34 @mock.patch(
35 35 'vcsserver.hgpatches.patch_largefiles_capabilities',
36 36 mock.Mock(side_effect=Exception("Must not be called")))
37 37 def test_applies_largefiles_patch_only_if_mercurial_is_available():
38 38 main.main([])
39 39
40 40
41 41 @pytest.mark.parametrize('given, expected', [
42 42 ('bad', 'bad'),
43 43 ('query&foo=bar', 'query&foo=bar'),
44 44 ('equery&auth_token=bar', 'equery&auth_token=*****'),
45 45 ('a;b;c;query&foo=bar&auth_token=secret',
46 46 'a&b&c&query&foo=bar&auth_token=*****'),
47 47 ('', ''),
48 48 (None, None),
49 49 ('foo=bar', 'foo=bar'),
50 50 ('auth_token=secret', 'auth_token=*****'),
51 51 ('auth_token=secret&api_key=secret2',
52 52 'auth_token=*****&api_key=*****'),
53 53 ('auth_token=secret&api_key=secret2&param=value',
54 54 'auth_token=*****&api_key=*****&param=value'),
55 55 ])
56 56 def test_obfuscate_qs(given, expected):
57 57 assert expected == obfuscate_qs(given)
@@ -1,249 +1,249 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import io
19 19
20 20 import dulwich.protocol
21 21 import mock
22 22 import pytest
23 23 import webob
24 24 import webtest
25 25
26 26 from vcsserver import hooks, pygrack
27 27
28 28 # pylint: disable=redefined-outer-name,protected-access
29 29
30 30
31 31 @pytest.fixture()
32 32 def pygrack_instance(tmpdir):
33 33 """
34 34 Creates a pygrack app instance.
35 35
36 36 Right now, it does not much helpful regarding the passed directory.
37 37 It just contains the required folders to pass the signature test.
38 38 """
39 39 for dir_name in ('config', 'head', 'info', 'objects', 'refs'):
40 40 tmpdir.mkdir(dir_name)
41 41
42 42 return pygrack.GitRepository('repo_name', str(tmpdir), 'git', False, {})
43 43
44 44
45 45 @pytest.fixture()
46 46 def pygrack_app(pygrack_instance):
47 47 """
48 48 Creates a pygrack app wrapped in webtest.TestApp.
49 49 """
50 50 return webtest.TestApp(pygrack_instance)
51 51
52 52
53 53 def test_invalid_service_info_refs_returns_403(pygrack_app):
54 54 response = pygrack_app.get('/info/refs?service=git-upload-packs',
55 55 expect_errors=True)
56 56
57 57 assert response.status_int == 403
58 58
59 59
60 60 def test_invalid_endpoint_returns_403(pygrack_app):
61 61 response = pygrack_app.post('/git-upload-packs', expect_errors=True)
62 62
63 63 assert response.status_int == 403
64 64
65 65
66 66 @pytest.mark.parametrize('sideband', [
67 67 'side-band-64k',
68 68 'side-band',
69 69 'side-band no-progress',
70 70 ])
71 71 def test_pre_pull_hook_fails_with_sideband(pygrack_app, sideband):
72 72 request = ''.join([
73 73 '0054want 74730d410fcb6603ace96f1dc55ea6196122532d ',
74 74 'multi_ack %s ofs-delta\n' % sideband,
75 75 '0000',
76 76 '0009done\n',
77 77 ])
78 78 with mock.patch('vcsserver.hooks.git_pre_pull',
79 79 return_value=hooks.HookResponse(1, 'foo')):
80 80 response = pygrack_app.post(
81 81 '/git-upload-pack', params=request,
82 82 content_type='application/x-git-upload-pack')
83 83
84 84 data = io.BytesIO(response.body)
85 85 proto = dulwich.protocol.Protocol(data.read, None)
86 86 packets = list(proto.read_pkt_seq())
87 87
88 88 expected_packets = [
89 89 'NAK\n', '\x02foo', '\x02Pre pull hook failed: aborting\n',
90 90 '\x01' + pygrack.GitRepository.EMPTY_PACK,
91 91 ]
92 92 assert packets == expected_packets
93 93
94 94
95 95 def test_pre_pull_hook_fails_no_sideband(pygrack_app):
96 96 request = ''.join([
97 97 '0054want 74730d410fcb6603ace96f1dc55ea6196122532d ' +
98 98 'multi_ack ofs-delta\n'
99 99 '0000',
100 100 '0009done\n',
101 101 ])
102 102 with mock.patch('vcsserver.hooks.git_pre_pull',
103 103 return_value=hooks.HookResponse(1, 'foo')):
104 104 response = pygrack_app.post(
105 105 '/git-upload-pack', params=request,
106 106 content_type='application/x-git-upload-pack')
107 107
108 108 assert response.body == pygrack.GitRepository.EMPTY_PACK
109 109
110 110
111 111 def test_pull_has_hook_messages(pygrack_app):
112 112 request = ''.join([
113 113 '0054want 74730d410fcb6603ace96f1dc55ea6196122532d ' +
114 114 'multi_ack side-band-64k ofs-delta\n'
115 115 '0000',
116 116 '0009done\n',
117 117 ])
118 118 with mock.patch('vcsserver.hooks.git_pre_pull',
119 119 return_value=hooks.HookResponse(0, 'foo')):
120 120 with mock.patch('vcsserver.hooks.git_post_pull',
121 121 return_value=hooks.HookResponse(1, 'bar')):
122 122 with mock.patch('vcsserver.subprocessio.SubprocessIOChunker',
123 123 return_value=['0008NAK\n0009subp\n0000']):
124 124 response = pygrack_app.post(
125 125 '/git-upload-pack', params=request,
126 126 content_type='application/x-git-upload-pack')
127 127
128 128 data = io.BytesIO(response.body)
129 129 proto = dulwich.protocol.Protocol(data.read, None)
130 130 packets = list(proto.read_pkt_seq())
131 131
132 132 assert packets == ['NAK\n', '\x02foo', 'subp\n', '\x02bar']
133 133
134 134
135 135 def test_get_want_capabilities(pygrack_instance):
136 136 data = io.BytesIO(
137 137 '0054want 74730d410fcb6603ace96f1dc55ea6196122532d ' +
138 138 'multi_ack side-band-64k ofs-delta\n00000009done\n')
139 139
140 140 request = webob.Request({
141 141 'wsgi.input': data,
142 142 'REQUEST_METHOD': 'POST',
143 143 'webob.is_body_seekable': True
144 144 })
145 145
146 146 capabilities = pygrack_instance._get_want_capabilities(request)
147 147
148 148 assert capabilities == frozenset(
149 149 ('ofs-delta', 'multi_ack', 'side-band-64k'))
150 150 assert data.tell() == 0
151 151
152 152
153 153 @pytest.mark.parametrize('data,capabilities,expected', [
154 154 ('foo', [], []),
155 155 ('', ['side-band-64k'], []),
156 156 ('', ['side-band'], []),
157 157 ('foo', ['side-band-64k'], ['0008\x02foo']),
158 158 ('foo', ['side-band'], ['0008\x02foo']),
159 159 ('f'*1000, ['side-band-64k'], ['03ed\x02' + 'f' * 1000]),
160 160 ('f'*1000, ['side-band'], ['03e8\x02' + 'f' * 995, '000a\x02fffff']),
161 161 ('f'*65520, ['side-band-64k'], ['fff0\x02' + 'f' * 65515, '000a\x02fffff']),
162 162 ('f'*65520, ['side-band'], ['03e8\x02' + 'f' * 995] * 65 + ['0352\x02' + 'f' * 845]),
163 163 ], ids=[
164 164 'foo-empty',
165 165 'empty-64k', 'empty',
166 166 'foo-64k', 'foo',
167 167 'f-1000-64k', 'f-1000',
168 168 'f-65520-64k', 'f-65520'])
169 169 def test_get_messages(pygrack_instance, data, capabilities, expected):
170 170 messages = pygrack_instance._get_messages(data, capabilities)
171 171
172 172 assert messages == expected
173 173
174 174
175 175 @pytest.mark.parametrize('response,capabilities,pre_pull_messages,post_pull_messages', [
176 176 # Unexpected response
177 177 ('unexpected_response', ['side-band-64k'], 'foo', 'bar'),
178 178 # No sideband
179 179 ('no-sideband', [], 'foo', 'bar'),
180 180 # No messages
181 181 ('no-messages', ['side-band-64k'], '', ''),
182 182 ])
183 183 def test_inject_messages_to_response_nothing_to_do(
184 184 pygrack_instance, response, capabilities, pre_pull_messages,
185 185 post_pull_messages):
186 186 new_response = pygrack_instance._inject_messages_to_response(
187 187 response, capabilities, pre_pull_messages, post_pull_messages)
188 188
189 189 assert new_response == response
190 190
191 191
192 192 @pytest.mark.parametrize('capabilities', [
193 193 ['side-band'],
194 194 ['side-band-64k'],
195 195 ])
196 196 def test_inject_messages_to_response_single_element(pygrack_instance,
197 197 capabilities):
198 198 response = ['0008NAK\n0009subp\n0000']
199 199 new_response = pygrack_instance._inject_messages_to_response(
200 200 response, capabilities, 'foo', 'bar')
201 201
202 202 expected_response = [
203 203 '0008NAK\n', '0008\x02foo', '0009subp\n', '0008\x02bar', '0000']
204 204
205 205 assert new_response == expected_response
206 206
207 207
208 208 @pytest.mark.parametrize('capabilities', [
209 209 ['side-band'],
210 210 ['side-band-64k'],
211 211 ])
212 212 def test_inject_messages_to_response_multi_element(pygrack_instance,
213 213 capabilities):
214 214 response = [
215 215 '0008NAK\n000asubp1\n', '000asubp2\n', '000asubp3\n', '000asubp4\n0000']
216 216 new_response = pygrack_instance._inject_messages_to_response(
217 217 response, capabilities, 'foo', 'bar')
218 218
219 219 expected_response = [
220 220 '0008NAK\n', '0008\x02foo', '000asubp1\n', '000asubp2\n', '000asubp3\n',
221 221 '000asubp4\n', '0008\x02bar', '0000'
222 222 ]
223 223
224 224 assert new_response == expected_response
225 225
226 226
227 227 def test_build_failed_pre_pull_response_no_sideband(pygrack_instance):
228 228 response = pygrack_instance._build_failed_pre_pull_response([], 'foo')
229 229
230 230 assert response == [pygrack.GitRepository.EMPTY_PACK]
231 231
232 232
233 233 @pytest.mark.parametrize('capabilities', [
234 234 ['side-band'],
235 235 ['side-band-64k'],
236 236 ['side-band-64k', 'no-progress'],
237 237 ])
238 238 def test_build_failed_pre_pull_response(pygrack_instance, capabilities):
239 239 response = pygrack_instance._build_failed_pre_pull_response(
240 240 capabilities, 'foo')
241 241
242 242 expected_response = [
243 243 '0008NAK\n', '0008\x02foo', '0024\x02Pre pull hook failed: aborting\n',
244 244 '%04x\x01%s' % (len(pygrack.GitRepository.EMPTY_PACK) + 5,
245 245 pygrack.GitRepository.EMPTY_PACK),
246 246 '0000',
247 247 ]
248 248
249 249 assert response == expected_response
@@ -1,86 +1,86 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import os
19 19
20 20 import mercurial.hg
21 21 import mercurial.ui
22 22 import mercurial.error
23 23 import mock
24 24 import pytest
25 25 import webtest
26 26
27 27 from vcsserver import scm_app
28 28
29 29
30 30 def test_hg_does_not_accept_invalid_cmd(tmpdir):
31 31 repo = mercurial.hg.repository(mercurial.ui.ui(), str(tmpdir), create=True)
32 32 app = webtest.TestApp(scm_app.HgWeb(repo))
33 33
34 34 response = app.get('/repo?cmd=invalidcmd', expect_errors=True)
35 35
36 36 assert response.status_int == 400
37 37
38 38
39 39 def test_create_hg_wsgi_app_requirement_error(tmpdir):
40 40 repo = mercurial.hg.repository(mercurial.ui.ui(), str(tmpdir), create=True)
41 41 config = (
42 42 ('paths', 'default', ''),
43 43 )
44 44 with mock.patch('vcsserver.scm_app.HgWeb') as hgweb_mock:
45 45 hgweb_mock.side_effect = mercurial.error.RequirementError()
46 46 with pytest.raises(Exception):
47 47 scm_app.create_hg_wsgi_app(str(tmpdir), repo, config)
48 48
49 49
50 50 def test_git_returns_not_found(tmpdir):
51 51 app = webtest.TestApp(
52 52 scm_app.GitHandler(str(tmpdir), 'repo_name', 'git', False, {}))
53 53
54 54 response = app.get('/repo_name/inforefs?service=git-upload-pack',
55 55 expect_errors=True)
56 56
57 57 assert response.status_int == 404
58 58
59 59
60 60 def test_git(tmpdir):
61 61 for dir_name in ('config', 'head', 'info', 'objects', 'refs'):
62 62 tmpdir.mkdir(dir_name)
63 63
64 64 app = webtest.TestApp(
65 65 scm_app.GitHandler(str(tmpdir), 'repo_name', 'git', False, {}))
66 66
67 67 # We set service to git-upload-packs to trigger a 403
68 68 response = app.get('/repo_name/inforefs?service=git-upload-packs',
69 69 expect_errors=True)
70 70
71 71 assert response.status_int == 403
72 72
73 73
74 74 def test_git_fallbacks_to_git_folder(tmpdir):
75 75 tmpdir.mkdir('.git')
76 76 for dir_name in ('config', 'head', 'info', 'objects', 'refs'):
77 77 tmpdir.mkdir(os.path.join('.git', dir_name))
78 78
79 79 app = webtest.TestApp(
80 80 scm_app.GitHandler(str(tmpdir), 'repo_name', 'git', False, {}))
81 81
82 82 # We set service to git-upload-packs to trigger a 403
83 83 response = app.get('/repo_name/inforefs?service=git-upload-packs',
84 84 expect_errors=True)
85 85
86 86 assert response.status_int == 403
@@ -1,39 +1,39 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import os
19 19
20 20 import mock
21 21 import pytest
22 22
23 23 from vcsserver.server import VcsServer
24 24
25 25
26 26 def test_provides_the_pid(server):
27 27 pid = server.get_pid()
28 28 assert pid == os.getpid()
29 29
30 30
31 31 def test_allows_to_trigger_the_garbage_collector(server):
32 32 with mock.patch('gc.collect') as collect:
33 33 server.run_gc()
34 34 assert collect.called
35 35
36 36
37 37 @pytest.fixture
38 38 def server():
39 39 return VcsServer()
@@ -1,122 +1,122 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import io
19 19 import os
20 20 import sys
21 21
22 22 import pytest
23 23
24 24 from vcsserver import subprocessio
25 25
26 26
27 27 @pytest.fixture(scope='module')
28 28 def environ():
29 29 """Delete coverage variables, as they make the tests fail."""
30 30 env = dict(os.environ)
31 31 for key in env.keys():
32 32 if key.startswith('COV_CORE_'):
33 33 del env[key]
34 34
35 35 return env
36 36
37 37
38 38 def _get_python_args(script):
39 39 return [sys.executable, '-c',
40 40 'import sys; import time; import shutil; ' + script]
41 41
42 42
43 43 def test_raise_exception_on_non_zero_return_code(environ):
44 44 args = _get_python_args('sys.exit(1)')
45 45 with pytest.raises(EnvironmentError):
46 46 list(subprocessio.SubprocessIOChunker(args, shell=False, env=environ))
47 47
48 48
49 49 def test_does_not_fail_on_non_zero_return_code(environ):
50 50 args = _get_python_args('sys.exit(1)')
51 51 output = ''.join(subprocessio.SubprocessIOChunker(
52 52 args, shell=False, fail_on_return_code=False, env=environ))
53 53
54 54 assert output == ''
55 55
56 56
57 57 def test_raise_exception_on_stderr(environ):
58 58 args = _get_python_args('sys.stderr.write("X"); time.sleep(1);')
59 59 with pytest.raises(EnvironmentError) as excinfo:
60 60 list(subprocessio.SubprocessIOChunker(args, shell=False, env=environ))
61 61
62 62 assert 'exited due to an error:\nX' in str(excinfo.value)
63 63
64 64
65 65 def test_does_not_fail_on_stderr(environ):
66 66 args = _get_python_args('sys.stderr.write("X"); time.sleep(1);')
67 67 output = ''.join(subprocessio.SubprocessIOChunker(
68 68 args, shell=False, fail_on_stderr=False, env=environ))
69 69
70 70 assert output == ''
71 71
72 72
73 73 @pytest.mark.parametrize('size', [1, 10**5])
74 74 def test_output_with_no_input(size, environ):
75 75 print type(environ)
76 76 data = 'X'
77 77 args = _get_python_args('sys.stdout.write("%s" * %d)' % (data, size))
78 78 output = ''.join(subprocessio.SubprocessIOChunker(
79 79 args, shell=False, env=environ))
80 80
81 81 assert output == data * size
82 82
83 83
84 84 @pytest.mark.parametrize('size', [1, 10**5])
85 85 def test_output_with_no_input_does_not_fail(size, environ):
86 86 data = 'X'
87 87 args = _get_python_args(
88 88 'sys.stdout.write("%s" * %d); sys.exit(1)' % (data, size))
89 89 output = ''.join(subprocessio.SubprocessIOChunker(
90 90 args, shell=False, fail_on_return_code=False, env=environ))
91 91
92 92 print len(data * size), len(output)
93 93 assert output == data * size
94 94
95 95
96 96 @pytest.mark.parametrize('size', [1, 10**5])
97 97 def test_output_with_input(size, environ):
98 98 data = 'X' * size
99 99 inputstream = io.BytesIO(data)
100 100 # This acts like the cat command.
101 101 args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)')
102 102 output = ''.join(subprocessio.SubprocessIOChunker(
103 103 args, shell=False, inputstream=inputstream, env=environ))
104 104
105 105 print len(data), len(output)
106 106 assert output == data
107 107
108 108
109 109 @pytest.mark.parametrize('size', [1, 10**5])
110 110 def test_output_with_input_skipping_iterator(size, environ):
111 111 data = 'X' * size
112 112 inputstream = io.BytesIO(data)
113 113 # This acts like the cat command.
114 114 args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)')
115 115
116 116 # Note: assigning the chunker makes sure that it is not deleted too early
117 117 chunker = subprocessio.SubprocessIOChunker(
118 118 args, shell=False, inputstream=inputstream, env=environ)
119 119 output = ''.join(chunker.output)
120 120
121 121 print len(data), len(output)
122 122 assert output == data
@@ -1,67 +1,67 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import io
19 19 import mock
20 20 import pytest
21 21 import sys
22 22
23 23
24 24 class MockPopen(object):
25 25 def __init__(self, stderr):
26 26 self.stdout = io.BytesIO('')
27 27 self.stderr = io.BytesIO(stderr)
28 28 self.returncode = 1
29 29
30 30 def wait(self):
31 31 pass
32 32
33 33
34 34 INVALID_CERTIFICATE_STDERR = '\n'.join([
35 35 'svnrdump: E230001: Unable to connect to a repository at URL url',
36 36 'svnrdump: E230001: Server SSL certificate verification failed: issuer is not trusted',
37 37 ])
38 38
39 39
40 40 @pytest.mark.parametrize('stderr,expected_reason', [
41 41 (INVALID_CERTIFICATE_STDERR, 'INVALID_CERTIFICATE'),
42 42 ('svnrdump: E123456', 'UNKNOWN'),
43 ])
43 ], ids=['invalid-cert-stderr', 'svnrdump-err-123456'])
44 44 @pytest.mark.xfail(sys.platform == "cygwin",
45 45 reason="SVN not packaged for Cygwin")
46 46 def test_import_remote_repository_certificate_error(stderr, expected_reason):
47 47 from vcsserver import svn
48 48
49 49 remote = svn.SvnRemote(None)
50 50 remote.is_path_valid_repository = lambda wire, path: True
51 51
52 52 with mock.patch('subprocess.Popen',
53 53 return_value=MockPopen(stderr)):
54 54 with pytest.raises(Exception) as excinfo:
55 55 remote.import_remote_repository({'path': 'path'}, 'url')
56 56
57 57 expected_error_args = (
58 58 'Failed to dump the remote repository from url.',
59 59 expected_reason)
60 60
61 61 assert excinfo.value.args == expected_error_args
62 62
63 63
64 64 def test_svn_libraries_can_be_imported():
65 65 import svn
66 66 import svn.client
67 67 assert svn.client is not None
@@ -1,132 +1,132 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import subprocess
19 19 import StringIO
20 20 import time
21 21
22 22 import pytest
23 23
24 from fixture import TestINI
24 from fixture import ContextINI
25 25
26 26
27 27 @pytest.mark.parametrize("arguments, expected_texts", [
28 28 (['--threadpool=192'], [
29 29 'threadpool_size: 192',
30 30 'worker pool of size 192 created',
31 31 'Threadpool size set to 192']),
32 32 (['--locale=fake'], [
33 33 'Cannot set locale, not configuring the locale system']),
34 34 (['--timeout=5'], [
35 35 'Timeout for RPC calls set to 5.0 seconds']),
36 36 (['--log-level=info'], [
37 37 'log_level: info']),
38 38 (['--port={port}'], [
39 39 'port: {port}',
40 40 'created daemon on localhost:{port}']),
41 41 (['--host=127.0.0.1', '--port={port}'], [
42 42 'port: {port}',
43 43 'host: 127.0.0.1',
44 44 'created daemon on 127.0.0.1:{port}']),
45 45 (['--config=/bad/file'], ['OSError: File /bad/file does not exist']),
46 46 ])
47 47 def test_vcsserver_calls(arguments, expected_texts, vcsserver_port):
48 48 port_argument = '--port={port}'
49 49 if port_argument not in arguments:
50 50 arguments.append(port_argument)
51 51 arguments = _replace_port(arguments, vcsserver_port)
52 52 expected_texts = _replace_port(expected_texts, vcsserver_port)
53 53 output = call_vcs_server_with_arguments(arguments)
54 54 for text in expected_texts:
55 55 assert text in output
56 56
57 57
58 58 def _replace_port(values, port):
59 59 return [value.format(port=port) for value in values]
60 60
61 61
62 62 def test_vcsserver_with_config(vcsserver_port):
63 63 ini_def = [
64 64 {'DEFAULT': {'host': '127.0.0.1'}},
65 65 {'DEFAULT': {'threadpool_size': '111'}},
66 66 {'DEFAULT': {'port': vcsserver_port}},
67 67 ]
68 68
69 with TestINI('test.ini', ini_def) as new_test_ini_path:
69 with ContextINI('test.ini', ini_def) as new_test_ini_path:
70 70 output = call_vcs_server_with_arguments(
71 71 ['--config=' + new_test_ini_path])
72 72
73 73 expected_texts = [
74 74 'host: 127.0.0.1',
75 75 'Threadpool size set to 111',
76 76 ]
77 77 for text in expected_texts:
78 78 assert text in output
79 79
80 80
81 81 def test_vcsserver_with_config_cli_overwrite(vcsserver_port):
82 82 ini_def = [
83 83 {'DEFAULT': {'host': '127.0.0.1'}},
84 84 {'DEFAULT': {'port': vcsserver_port}},
85 85 {'DEFAULT': {'threadpool_size': '111'}},
86 86 {'DEFAULT': {'timeout': '0'}},
87 87 ]
88 with TestINI('test.ini', ini_def) as new_test_ini_path:
88 with ContextINI('test.ini', ini_def) as new_test_ini_path:
89 89 output = call_vcs_server_with_arguments([
90 90 '--config=' + new_test_ini_path,
91 91 '--host=128.0.0.1',
92 92 '--threadpool=256',
93 93 '--timeout=5'])
94 94 expected_texts = [
95 95 'host: 128.0.0.1',
96 96 'Threadpool size set to 256',
97 97 'Timeout for RPC calls set to 5.0 seconds',
98 98 ]
99 99 for text in expected_texts:
100 100 assert text in output
101 101
102 102
103 103 def call_vcs_server_with_arguments(args):
104 104 vcs = subprocess.Popen(
105 105 ["vcsserver"] + args,
106 106 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
107 107
108 108 output = read_output_until(
109 109 "Starting vcsserver.main", vcs.stdout)
110 110 vcs.terminate()
111 111 return output
112 112
113 113
114 114 def call_vcs_server_with_non_existing_config_file(args):
115 115 vcs = subprocess.Popen(
116 116 ["vcsserver", "--config=/tmp/bad"] + args,
117 117 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
118 118 output = read_output_until(
119 119 "Starting vcsserver.main", vcs.stdout)
120 120 vcs.terminate()
121 121 return output
122 122
123 123
124 124 def read_output_until(expected, source, timeout=5):
125 125 ts = time.time()
126 126 buf = StringIO.StringIO()
127 127 while time.time() - ts < timeout:
128 128 line = source.readline()
129 129 buf.write(line)
130 130 if expected in line:
131 131 break
132 132 return buf.getvalue()
@@ -1,96 +1,96 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import wsgiref.simple_server
19 19 import wsgiref.validate
20 20
21 21 from vcsserver import wsgi_app_caller
22 22
23 23
24 24 # pylint: disable=protected-access,too-many-public-methods
25 25
26 26
27 27 @wsgiref.validate.validator
28 28 def demo_app(environ, start_response):
29 29 """WSGI app used for testing."""
30 30 data = [
31 31 'Hello World!\n',
32 32 'input_data=%s\n' % environ['wsgi.input'].read(),
33 33 ]
34 34 for key, value in sorted(environ.items()):
35 35 data.append('%s=%s\n' % (key, value))
36 36
37 37 write = start_response("200 OK", [('Content-Type', 'text/plain')])
38 38 write('Old school write method\n')
39 39 write('***********************\n')
40 40 return data
41 41
42 42
43 43 BASE_ENVIRON = {
44 44 'REQUEST_METHOD': 'GET',
45 45 'SERVER_NAME': 'localhost',
46 46 'SERVER_PORT': '80',
47 47 'SCRIPT_NAME': '',
48 48 'PATH_INFO': '/',
49 49 'QUERY_STRING': '',
50 50 'foo.var': 'bla',
51 51 }
52 52
53 53
54 54 def test_complete_environ():
55 55 environ = dict(BASE_ENVIRON)
56 56 data = "data"
57 57 wsgi_app_caller._complete_environ(environ, data)
58 58 wsgiref.validate.check_environ(environ)
59 59
60 60 assert data == environ['wsgi.input'].read()
61 61
62 62
63 63 def test_start_response():
64 64 start_response = wsgi_app_caller._StartResponse()
65 65 status = '200 OK'
66 66 headers = [('Content-Type', 'text/plain')]
67 67 start_response(status, headers)
68 68
69 69 assert status == start_response.status
70 70 assert headers == start_response.headers
71 71
72 72
73 73 def test_start_response_with_error():
74 74 start_response = wsgi_app_caller._StartResponse()
75 75 status = '500 Internal Server Error'
76 76 headers = [('Content-Type', 'text/plain')]
77 77 start_response(status, headers, (None, None, None))
78 78
79 79 assert status == start_response.status
80 80 assert headers == start_response.headers
81 81
82 82
83 83 def test_wsgi_app_caller():
84 84 caller = wsgi_app_caller.WSGIAppCaller(demo_app)
85 85 environ = dict(BASE_ENVIRON)
86 86 input_data = 'some text'
87 87 responses, status, headers = caller.handle(environ, input_data)
88 88 response = ''.join(responses)
89 89
90 90 assert status == '200 OK'
91 91 assert headers == [('Content-Type', 'text/plain')]
92 92 assert response.startswith(
93 93 'Old school write method\n***********************\n')
94 94 assert 'Hello World!\n' in response
95 95 assert 'foo.var=bla\n' in response
96 96 assert 'input_data=%s\n' % input_data in response
@@ -1,57 +1,57 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18
19 19
20 20 # TODO: johbo: That's a copy from rhodecode
21 21 def safe_str(unicode_, to_encoding=['utf8']):
22 22 """
23 23 safe str function. Does few trick to turn unicode_ into string
24 24
25 25 In case of UnicodeEncodeError, we try to return it with encoding detected
26 26 by chardet library if it fails fallback to string with errors replaced
27 27
28 28 :param unicode_: unicode to encode
29 29 :rtype: str
30 30 :returns: str object
31 31 """
32 32
33 33 # if it's not basestr cast to str
34 34 if not isinstance(unicode_, basestring):
35 35 return str(unicode_)
36 36
37 37 if isinstance(unicode_, str):
38 38 return unicode_
39 39
40 40 if not isinstance(to_encoding, (list, tuple)):
41 41 to_encoding = [to_encoding]
42 42
43 43 for enc in to_encoding:
44 44 try:
45 45 return unicode_.encode(enc)
46 46 except UnicodeEncodeError:
47 47 pass
48 48
49 49 try:
50 50 import chardet
51 51 encoding = chardet.detect(unicode_)['encoding']
52 52 if encoding is None:
53 53 raise UnicodeEncodeError()
54 54
55 55 return unicode_.encode(encoding)
56 56 except (ImportError, UnicodeEncodeError):
57 57 return unicode_.encode(to_encoding[0], 'replace')
@@ -1,116 +1,116 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2016 RodeCode GmbH
2 # Copyright (C) 2014-2017 RodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 """Extract the responses of a WSGI app."""
19 19
20 20 __all__ = ('WSGIAppCaller',)
21 21
22 22 import io
23 23 import logging
24 24 import os
25 25
26 26
27 27 log = logging.getLogger(__name__)
28 28
29 29 DEV_NULL = open(os.devnull)
30 30
31 31
32 32 def _complete_environ(environ, input_data):
33 33 """Update the missing wsgi.* variables of a WSGI environment.
34 34
35 35 :param environ: WSGI environment to update
36 36 :type environ: dict
37 37 :param input_data: data to be read by the app
38 38 :type input_data: str
39 39 """
40 40 environ.update({
41 41 'wsgi.version': (1, 0),
42 42 'wsgi.url_scheme': 'http',
43 43 'wsgi.multithread': True,
44 44 'wsgi.multiprocess': True,
45 45 'wsgi.run_once': False,
46 46 'wsgi.input': io.BytesIO(input_data),
47 47 'wsgi.errors': DEV_NULL,
48 48 })
49 49
50 50
51 51 # pylint: disable=too-few-public-methods
52 52 class _StartResponse(object):
53 53 """Save the arguments of a start_response call."""
54 54
55 55 __slots__ = ['status', 'headers', 'content']
56 56
57 57 def __init__(self):
58 58 self.status = None
59 59 self.headers = None
60 60 self.content = []
61 61
62 62 def __call__(self, status, headers, exc_info=None):
63 63 # TODO(skreft): do something meaningful with the exc_info
64 64 exc_info = None # avoid dangling circular reference
65 65 self.status = status
66 66 self.headers = headers
67 67
68 68 return self.write
69 69
70 70 def write(self, content):
71 71 """Write method returning when calling this object.
72 72
73 73 All the data written is then available in content.
74 74 """
75 75 self.content.append(content)
76 76
77 77
78 78 class WSGIAppCaller(object):
79 79 """Calls a WSGI app."""
80 80
81 81 def __init__(self, app):
82 82 """
83 83 :param app: WSGI app to call
84 84 """
85 85 self.app = app
86 86
87 87 def handle(self, environ, input_data):
88 88 """Process a request with the WSGI app.
89 89
90 90 The returned data of the app is fully consumed into a list.
91 91
92 92 :param environ: WSGI environment to update
93 93 :type environ: dict
94 94 :param input_data: data to be read by the app
95 95 :type input_data: str
96 96
97 97 :returns: a tuple with the contents, status and headers
98 98 :rtype: (list<str>, str, list<(str, str)>)
99 99 """
100 100 _complete_environ(environ, input_data)
101 101 start_response = _StartResponse()
102 102 log.debug("Calling wrapped WSGI application")
103 103 responses = self.app(environ, start_response)
104 104 responses_list = list(responses)
105 105 existing_responses = start_response.content
106 106 if existing_responses:
107 107 log.debug(
108 108 "Adding returned response to response written via write()")
109 109 existing_responses.extend(responses_list)
110 110 responses_list = existing_responses
111 111 if hasattr(responses, 'close'):
112 112 log.debug("Closing iterator from WSGI application")
113 113 responses.close()
114 114
115 115 log.debug("Handling of WSGI request done, returning response")
116 116 return responses_list, start_response.status, start_response.headers
General Comments 0
You need to be logged in to leave comments. Login now