##// END OF EJS Templates
release: merge back stable branch into default
marcink -
r688:3cb906bd merge default
parent child Browse files
Show More
@@ -0,0 +1,15 b''
1 |RCE| 4.3.1 |RNS|
2 -----------------
3
4 Release Date
5 ^^^^^^^^^^^^
6
7 - 2016-08-23
8
9 Fixes
10 ^^^^^
11
12 - Core: fixed database session cleanups. This will make sure RhodeCode can
13 function correctly after database server problems. Fixes #4173, refs #4166
14 - Diffs: limit the file context to ~1mln lines. Fixes #4184, also make sure
15 this doesn't trigger Integer overflow for msgpack. No newline at end of file
@@ -1,8 +1,9 b''
1 1bd3e92b7e2e2d2024152b34bb88dff1db544a71 v4.0.0
1 1bd3e92b7e2e2d2024152b34bb88dff1db544a71 v4.0.0
2 170c5398320ea6cddd50955e88d408794c21d43a v4.0.1
2 170c5398320ea6cddd50955e88d408794c21d43a v4.0.1
3 c3fe200198f5aa34cf2e4066df2881a9cefe3704 v4.1.0
3 c3fe200198f5aa34cf2e4066df2881a9cefe3704 v4.1.0
4 7fd5c850745e2ea821fb4406af5f4bff9b0a7526 v4.1.1
4 7fd5c850745e2ea821fb4406af5f4bff9b0a7526 v4.1.1
5 41c87da28a179953df86061d817bc35533c66dd2 v4.1.2
5 41c87da28a179953df86061d817bc35533c66dd2 v4.1.2
6 baaf9f5bcea3bae0ef12ae20c8b270482e62abb6 v4.2.0
6 baaf9f5bcea3bae0ef12ae20c8b270482e62abb6 v4.2.0
7 32a70c7e56844a825f61df496ee5eaf8c3c4e189 v4.2.1
7 32a70c7e56844a825f61df496ee5eaf8c3c4e189 v4.2.1
8 fa695cdb411d294679ac081d595ac654e5613b03 v4.3.0
8 fa695cdb411d294679ac081d595ac654e5613b03 v4.3.0
9 0e4dc11b58cad833c513fe17bac39e6850edf959 v4.3.1
@@ -1,84 +1,128 b''
1 .. _svn-http:
1 .. _svn-http:
2
2
3 |svn| With Write Over HTTP
3 |svn| With Write Over HTTP
4 --------------------------
4 --------------------------
5
5
6 To use |svn| with write access, the currently supported method is over HTTP.
6 To use |svn| with read/write support over the |svn| protocol, you have to
7 This requires you to configure your local machine so that it can access your
7 configure HTTP |svn| backend.
8 |RCE| instance.
9
8
10 Prerequisites
9 Prerequisites
11 ^^^^^^^^^^^^^
10 ^^^^^^^^^^^^^
12
11
13 - Enable lab setting on your |RCE| instance, see :ref:`lab-settings`.
12 - Enable HTTP support inside labs setting on your |RCE| instance,
14 - You need to install the following tools on your local machine: ``Apache`` and
13 see :ref:`lab-settings`.
15 ``mod_dav_svn``. Use the following Ubuntu as an example.
14 - You need to install the following tools on the machine that is running an
15 instance of |RCE|:
16 ``Apache HTTP Server`` and
17 ``mod_dav_svn``.
18
19
20 Using Ubuntu Distribution as an example you can run:
16
21
17 .. code-block:: bash
22 .. code-block:: bash
18
23
19 $ sudo apt-get install apache2 libapache2-mod-svn
24 $ sudo apt-get install apache2 libapache2-mod-svn
20
25
21 Once installed you need to enable ``dav_svn``:
26 Once installed you need to enable ``dav_svn``:
22
27
23 .. code-block:: bash
28 .. code-block:: bash
24
29
25 $ sudo a2enmod dav_svn
30 $ sudo a2enmod dav_svn
26
31
27 Configuring Apache Setup
32 Configuring Apache Setup
28 ^^^^^^^^^^^^^^^^^^^^^^^^
33 ^^^^^^^^^^^^^^^^^^^^^^^^
29
34
30 .. tip::
35 .. tip::
31
36
32 It is recommended to run Apache on a port other than 80, due to possible
37 It is recommended to run Apache on a port other than 80, due to possible
33 conflicts with other HTTP servers like nginx. To do this, set the
38 conflicts with other HTTP servers like nginx. To do this, set the
34 ``Listen`` parameter in the ``/etc/apache2/ports.conf`` file, for example
39 ``Listen`` parameter in the ``/etc/apache2/ports.conf`` file, for example
35 ``Listen 8090``
40 ``Listen 8090``.
41
42
43 .. warning::
44
45 Make sure your Apache instance which runs the mod_dav_svn module is
46 only accessible by RhodeCode. Otherwise everyone is able to browse
47 the repositories or run subversion operations (checkout/commit/etc.).
36
48
37 It is also recommended to run apache as the same user as |RCE|, otherwise
49 It is also recommended to run apache as the same user as |RCE|, otherwise
38 permission issues could occur. To do this edit the ``/etc/apache2/envvars``
50 permission issues could occur. To do this edit the ``/etc/apache2/envvars``
39
51
40 .. code-block:: apache
52 .. code-block:: apache
41
53
42 export APACHE_RUN_USER=ubuntu
54 export APACHE_RUN_USER=rhodecode
43 export APACHE_RUN_GROUP=ubuntu
55 export APACHE_RUN_GROUP=rhodecode
44
56
45 1. To configure Apache, create and edit a virtual hosts file, for example
57 1. To configure Apache, create and edit a virtual hosts file, for example
46 :file:`/etc/apache2/sites-available/default.conf`, or create another
58 :file:`/etc/apache2/sites-available/default.conf`. Below is an example
47 virtual hosts file and add a location section inside the
59 how to use one with auto-generated config ```mod_dav_svn.conf```
48 ``<VirtualHost>`` section.
60 from configured |RCE| instance.
49
61
50 .. code-block:: apache
62 .. code-block:: apache
51
63
52 <Location />
64 <VirtualHost *:8080>
53 DAV svn
65 ServerAdmin rhodecode-admin@localhost
54 # Must be explicit path, relative not supported
66 DocumentRoot /var/www/html
55 SVNParentPath /PATH/TO/REPOSITORIES
67 ErrorLog ${'${APACHE_LOG_DIR}'}/error.log
56 SVNListParentPath On
68 CustomLog ${'${APACHE_LOG_DIR}'}/access.log combined
57 Allow from all
69 Include /home/user/.rccontrol/enterprise-1/mod_dav_svn.conf
58 Order allow,deny
70 </VirtualHost>
59 </Location>
60
71
61 .. note::
62
63 Once configured, check that you can see the list of repositories on your
64 |RCE| instance.
65
72
66 2. Go to the :menuselection:`Admin --> Settings --> Labs` page, and
73 2. Go to the :menuselection:`Admin --> Settings --> Labs` page, and
67 enable :guilabel:`Proxy Subversion HTTP requests`, and specify the
74 enable :guilabel:`Proxy Subversion HTTP requests`, and specify the
68 :guilabel:`Subversion HTTP Server URL`.
75 :guilabel:`Subversion HTTP Server URL`.
69
76
77 3. Open the |RCE| configuration file,
78 :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
79
80 4. Add the following configuration option in the ``[app:main]``
81 section if you don't have it yet.
82
83 This enable mapping of created |RCE| repo groups into special |svn| paths.
84 Each time a new repository group will be created the system will update
85 the template file, and create new mapping. Apache web server needs to be
86 reloaded to pick up the changes on this file.
87 It's recommended to add reload into a crontab so the changes can be picked
88 automatically once someone creates an repository group inside RhodeCode.
89
90
91 .. code-block:: ini
92
93 ##############################################
94 ### Subversion proxy support (mod_dav_svn) ###
95 ##############################################
96 ## Enable or disable the config file generation.
97 svn.proxy.generate_config = true
98 ## Generate config file with `SVNListParentPath` set to `On`.
99 svn.proxy.list_parent_path = true
100 ## Set location and file name of generated config file.
101 svn.proxy.config_file_path = %(here)s/mod_dav_svn.conf
102 ## File system path to the directory containing the repositories served by
103 ## RhodeCode.
104 svn.proxy.parent_path_root = /path/to/repo_store
105 ## Used as a prefix to the <Location> block in the generated config file. In
106 ## most cases it should be set to `/`.
107 svn.proxy.location_root = /
108
109
110 This would create a special template file called ```mod_dav_svn.conf```. We
111 used that file path in the apache config above inside the Include statement.
112
113
70 Using |svn|
114 Using |svn|
71 ^^^^^^^^^^^
115 ^^^^^^^^^^^
72
116
73 Once |svn| has been enabled on your instance, you can use it using the
117 Once |svn| has been enabled on your instance, you can use it using the
74 following examples. For more |svn| information, see the `Subversion Red Book`_
118 following examples. For more |svn| information, see the `Subversion Red Book`_
75
119
76 .. code-block:: bash
120 .. code-block:: bash
77
121
78 # To clone a repository
122 # To clone a repository
79 svn checkout http://my-svn-server.example.com/my-svn-repo
123 svn checkout http://my-svn-server.example.com/my-svn-repo
80
124
81 # svn commit
125 # svn commit
82 svn commit
126 svn commit
83
127
84 .. _Subversion Red Book: http://svnbook.red-bean.com/en/1.7/svn-book.html#svn.ref.svn
128 .. _Subversion Red Book: http://svnbook.red-bean.com/en/1.7/svn-book.html#svn.ref.svn
@@ -1,85 +1,86 b''
1 .. _rhodecode-release-notes-ref:
1 .. _rhodecode-release-notes-ref:
2
2
3 Release Notes
3 Release Notes
4 =============
4 =============
5
5
6 |RCE| 4.x Versions
6 |RCE| 4.x Versions
7 ------------------
7 ------------------
8
8
9 .. toctree::
9 .. toctree::
10 :maxdepth: 1
10 :maxdepth: 1
11
11
12 release-notes-4.3.1.rst
12 release-notes-4.3.0.rst
13 release-notes-4.3.0.rst
13 release-notes-4.2.1.rst
14 release-notes-4.2.1.rst
14 release-notes-4.2.0.rst
15 release-notes-4.2.0.rst
15 release-notes-4.1.2.rst
16 release-notes-4.1.2.rst
16 release-notes-4.1.1.rst
17 release-notes-4.1.1.rst
17 release-notes-4.1.0.rst
18 release-notes-4.1.0.rst
18 release-notes-4.0.1.rst
19 release-notes-4.0.1.rst
19 release-notes-4.0.0.rst
20 release-notes-4.0.0.rst
20
21
21 |RCE| 3.x Versions
22 |RCE| 3.x Versions
22 ------------------
23 ------------------
23
24
24 .. toctree::
25 .. toctree::
25 :maxdepth: 1
26 :maxdepth: 1
26
27
27 release-notes-3.8.4.rst
28 release-notes-3.8.4.rst
28 release-notes-3.8.3.rst
29 release-notes-3.8.3.rst
29 release-notes-3.8.2.rst
30 release-notes-3.8.2.rst
30 release-notes-3.8.1.rst
31 release-notes-3.8.1.rst
31 release-notes-3.8.0.rst
32 release-notes-3.8.0.rst
32 release-notes-3.7.1.rst
33 release-notes-3.7.1.rst
33 release-notes-3.7.0.rst
34 release-notes-3.7.0.rst
34 release-notes-3.6.1.rst
35 release-notes-3.6.1.rst
35 release-notes-3.6.0.rst
36 release-notes-3.6.0.rst
36 release-notes-3.5.2.rst
37 release-notes-3.5.2.rst
37 release-notes-3.5.1.rst
38 release-notes-3.5.1.rst
38 release-notes-3.5.0.rst
39 release-notes-3.5.0.rst
39 release-notes-3.4.1.rst
40 release-notes-3.4.1.rst
40 release-notes-3.4.0.rst
41 release-notes-3.4.0.rst
41 release-notes-3.3.4.rst
42 release-notes-3.3.4.rst
42 release-notes-3.3.3.rst
43 release-notes-3.3.3.rst
43 release-notes-3.3.2.rst
44 release-notes-3.3.2.rst
44 release-notes-3.3.1.rst
45 release-notes-3.3.1.rst
45 release-notes-3.3.0.rst
46 release-notes-3.3.0.rst
46 release-notes-3.2.3.rst
47 release-notes-3.2.3.rst
47 release-notes-3.2.2.rst
48 release-notes-3.2.2.rst
48 release-notes-3.2.1.rst
49 release-notes-3.2.1.rst
49 release-notes-3.2.0.rst
50 release-notes-3.2.0.rst
50 release-notes-3.1.1.rst
51 release-notes-3.1.1.rst
51 release-notes-3.1.0.rst
52 release-notes-3.1.0.rst
52 release-notes-3.0.2.rst
53 release-notes-3.0.2.rst
53 release-notes-3.0.1.rst
54 release-notes-3.0.1.rst
54 release-notes-3.0.0.rst
55 release-notes-3.0.0.rst
55
56
56 |RCE| 2.x Versions
57 |RCE| 2.x Versions
57 ------------------
58 ------------------
58
59
59 .. toctree::
60 .. toctree::
60 :maxdepth: 1
61 :maxdepth: 1
61
62
62 release-notes-2.2.8.rst
63 release-notes-2.2.8.rst
63 release-notes-2.2.7.rst
64 release-notes-2.2.7.rst
64 release-notes-2.2.6.rst
65 release-notes-2.2.6.rst
65 release-notes-2.2.5.rst
66 release-notes-2.2.5.rst
66 release-notes-2.2.4.rst
67 release-notes-2.2.4.rst
67 release-notes-2.2.3.rst
68 release-notes-2.2.3.rst
68 release-notes-2.2.2.rst
69 release-notes-2.2.2.rst
69 release-notes-2.2.1.rst
70 release-notes-2.2.1.rst
70 release-notes-2.2.0.rst
71 release-notes-2.2.0.rst
71 release-notes-2.1.0.rst
72 release-notes-2.1.0.rst
72 release-notes-2.0.2.rst
73 release-notes-2.0.2.rst
73 release-notes-2.0.1.rst
74 release-notes-2.0.1.rst
74 release-notes-2.0.0.rst
75 release-notes-2.0.0.rst
75
76
76 |RCE| 1.x Versions
77 |RCE| 1.x Versions
77 ------------------
78 ------------------
78
79
79 .. toctree::
80 .. toctree::
80 :maxdepth: 1
81 :maxdepth: 1
81
82
82 release-notes-1.7.2.rst
83 release-notes-1.7.2.rst
83 release-notes-1.7.1.rst
84 release-notes-1.7.1.rst
84 release-notes-1.7.0.rst
85 release-notes-1.7.0.rst
85 release-notes-1.6.0.rst
86 release-notes-1.6.0.rst
@@ -1,481 +1,504 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2016 RhodeCode GmbH
3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 Pylons middleware initialization
22 Pylons middleware initialization
23 """
23 """
24 import logging
24 import logging
25 from collections import OrderedDict
25 from collections import OrderedDict
26
26
27 from paste.registry import RegistryManager
27 from paste.registry import RegistryManager
28 from paste.gzipper import make_gzip_middleware
28 from paste.gzipper import make_gzip_middleware
29 from pylons.wsgiapp import PylonsApp
29 from pylons.wsgiapp import PylonsApp
30 from pyramid.authorization import ACLAuthorizationPolicy
30 from pyramid.authorization import ACLAuthorizationPolicy
31 from pyramid.config import Configurator
31 from pyramid.config import Configurator
32 from pyramid.settings import asbool, aslist
32 from pyramid.settings import asbool, aslist
33 from pyramid.wsgi import wsgiapp
33 from pyramid.wsgi import wsgiapp
34 from pyramid.httpexceptions import HTTPError, HTTPInternalServerError, HTTPFound
34 from pyramid.httpexceptions import HTTPError, HTTPInternalServerError, HTTPFound
35 from pyramid.events import ApplicationCreated
35 from pyramid.events import ApplicationCreated
36 import pyramid.httpexceptions as httpexceptions
36 import pyramid.httpexceptions as httpexceptions
37 from pyramid.renderers import render_to_response
37 from pyramid.renderers import render_to_response
38 from routes.middleware import RoutesMiddleware
38 from routes.middleware import RoutesMiddleware
39 import routes.util
39 import routes.util
40
40
41 import rhodecode
41 import rhodecode
42 from rhodecode.model import meta
42 from rhodecode.config import patches
43 from rhodecode.config import patches
43 from rhodecode.config.routing import STATIC_FILE_PREFIX
44 from rhodecode.config.routing import STATIC_FILE_PREFIX
44 from rhodecode.config.environment import (
45 from rhodecode.config.environment import (
45 load_environment, load_pyramid_environment)
46 load_environment, load_pyramid_environment)
46 from rhodecode.lib.exceptions import VCSServerUnavailable
47 from rhodecode.lib.exceptions import VCSServerUnavailable
47 from rhodecode.lib.vcs.exceptions import VCSCommunicationError
48 from rhodecode.lib.vcs.exceptions import VCSCommunicationError
48 from rhodecode.lib.middleware import csrf
49 from rhodecode.lib.middleware import csrf
49 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
50 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
50 from rhodecode.lib.middleware.https_fixup import HttpsFixup
51 from rhodecode.lib.middleware.https_fixup import HttpsFixup
51 from rhodecode.lib.middleware.vcs import VCSMiddleware
52 from rhodecode.lib.middleware.vcs import VCSMiddleware
52 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
53 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
53 from rhodecode.lib.utils2 import aslist as rhodecode_aslist
54 from rhodecode.lib.utils2 import aslist as rhodecode_aslist
54 from rhodecode.subscribers import scan_repositories_if_enabled
55 from rhodecode.subscribers import scan_repositories_if_enabled
55
56
56
57
57 log = logging.getLogger(__name__)
58 log = logging.getLogger(__name__)
58
59
59
60
60 # this is used to avoid avoid the route lookup overhead in routesmiddleware
61 # this is used to avoid avoid the route lookup overhead in routesmiddleware
61 # for certain routes which won't go to pylons to - eg. static files, debugger
62 # for certain routes which won't go to pylons to - eg. static files, debugger
62 # it is only needed for the pylons migration and can be removed once complete
63 # it is only needed for the pylons migration and can be removed once complete
63 class SkippableRoutesMiddleware(RoutesMiddleware):
64 class SkippableRoutesMiddleware(RoutesMiddleware):
64 """ Routes middleware that allows you to skip prefixes """
65 """ Routes middleware that allows you to skip prefixes """
65
66
66 def __init__(self, *args, **kw):
67 def __init__(self, *args, **kw):
67 self.skip_prefixes = kw.pop('skip_prefixes', [])
68 self.skip_prefixes = kw.pop('skip_prefixes', [])
68 super(SkippableRoutesMiddleware, self).__init__(*args, **kw)
69 super(SkippableRoutesMiddleware, self).__init__(*args, **kw)
69
70
70 def __call__(self, environ, start_response):
71 def __call__(self, environ, start_response):
71 for prefix in self.skip_prefixes:
72 for prefix in self.skip_prefixes:
72 if environ['PATH_INFO'].startswith(prefix):
73 if environ['PATH_INFO'].startswith(prefix):
73 # added to avoid the case when a missing /_static route falls
74 # added to avoid the case when a missing /_static route falls
74 # through to pylons and causes an exception as pylons is
75 # through to pylons and causes an exception as pylons is
75 # expecting wsgiorg.routingargs to be set in the environ
76 # expecting wsgiorg.routingargs to be set in the environ
76 # by RoutesMiddleware.
77 # by RoutesMiddleware.
77 if 'wsgiorg.routing_args' not in environ:
78 if 'wsgiorg.routing_args' not in environ:
78 environ['wsgiorg.routing_args'] = (None, {})
79 environ['wsgiorg.routing_args'] = (None, {})
79 return self.app(environ, start_response)
80 return self.app(environ, start_response)
80
81
81 return super(SkippableRoutesMiddleware, self).__call__(
82 return super(SkippableRoutesMiddleware, self).__call__(
82 environ, start_response)
83 environ, start_response)
83
84
84
85
85 def make_app(global_conf, static_files=True, **app_conf):
86 def make_app(global_conf, static_files=True, **app_conf):
86 """Create a Pylons WSGI application and return it
87 """Create a Pylons WSGI application and return it
87
88
88 ``global_conf``
89 ``global_conf``
89 The inherited configuration for this application. Normally from
90 The inherited configuration for this application. Normally from
90 the [DEFAULT] section of the Paste ini file.
91 the [DEFAULT] section of the Paste ini file.
91
92
92 ``app_conf``
93 ``app_conf``
93 The application's local configuration. Normally specified in
94 The application's local configuration. Normally specified in
94 the [app:<name>] section of the Paste ini file (where <name>
95 the [app:<name>] section of the Paste ini file (where <name>
95 defaults to main).
96 defaults to main).
96
97
97 """
98 """
98 # Apply compatibility patches
99 # Apply compatibility patches
99 patches.kombu_1_5_1_python_2_7_11()
100 patches.kombu_1_5_1_python_2_7_11()
100 patches.inspect_getargspec()
101 patches.inspect_getargspec()
101
102
102 # Configure the Pylons environment
103 # Configure the Pylons environment
103 config = load_environment(global_conf, app_conf)
104 config = load_environment(global_conf, app_conf)
104
105
105 # The Pylons WSGI app
106 # The Pylons WSGI app
106 app = PylonsApp(config=config)
107 app = PylonsApp(config=config)
107 if rhodecode.is_test:
108 if rhodecode.is_test:
108 app = csrf.CSRFDetector(app)
109 app = csrf.CSRFDetector(app)
109
110
110 expected_origin = config.get('expected_origin')
111 expected_origin = config.get('expected_origin')
111 if expected_origin:
112 if expected_origin:
112 # The API can be accessed from other Origins.
113 # The API can be accessed from other Origins.
113 app = csrf.OriginChecker(app, expected_origin,
114 app = csrf.OriginChecker(app, expected_origin,
114 skip_urls=[routes.util.url_for('api')])
115 skip_urls=[routes.util.url_for('api')])
115
116
116 # Establish the Registry for this application
117 # Establish the Registry for this application
117 app = RegistryManager(app)
118 app = RegistryManager(app)
118
119
119 app.config = config
120 app.config = config
120
121
121 return app
122 return app
122
123
123
124
124 def make_pyramid_app(global_config, **settings):
125 def make_pyramid_app(global_config, **settings):
125 """
126 """
126 Constructs the WSGI application based on Pyramid and wraps the Pylons based
127 Constructs the WSGI application based on Pyramid and wraps the Pylons based
127 application.
128 application.
128
129
129 Specials:
130 Specials:
130
131
131 * We migrate from Pylons to Pyramid. While doing this, we keep both
132 * We migrate from Pylons to Pyramid. While doing this, we keep both
132 frameworks functional. This involves moving some WSGI middlewares around
133 frameworks functional. This involves moving some WSGI middlewares around
133 and providing access to some data internals, so that the old code is
134 and providing access to some data internals, so that the old code is
134 still functional.
135 still functional.
135
136
136 * The application can also be integrated like a plugin via the call to
137 * The application can also be integrated like a plugin via the call to
137 `includeme`. This is accompanied with the other utility functions which
138 `includeme`. This is accompanied with the other utility functions which
138 are called. Changing this should be done with great care to not break
139 are called. Changing this should be done with great care to not break
139 cases when these fragments are assembled from another place.
140 cases when these fragments are assembled from another place.
140
141
141 """
142 """
142 # The edition string should be available in pylons too, so we add it here
143 # The edition string should be available in pylons too, so we add it here
143 # before copying the settings.
144 # before copying the settings.
144 settings.setdefault('rhodecode.edition', 'Community Edition')
145 settings.setdefault('rhodecode.edition', 'Community Edition')
145
146
146 # As long as our Pylons application does expect "unprepared" settings, make
147 # As long as our Pylons application does expect "unprepared" settings, make
147 # sure that we keep an unmodified copy. This avoids unintentional change of
148 # sure that we keep an unmodified copy. This avoids unintentional change of
148 # behavior in the old application.
149 # behavior in the old application.
149 settings_pylons = settings.copy()
150 settings_pylons = settings.copy()
150
151
151 sanitize_settings_and_apply_defaults(settings)
152 sanitize_settings_and_apply_defaults(settings)
152 config = Configurator(settings=settings)
153 config = Configurator(settings=settings)
153 add_pylons_compat_data(config.registry, global_config, settings_pylons)
154 add_pylons_compat_data(config.registry, global_config, settings_pylons)
154
155
155 load_pyramid_environment(global_config, settings)
156 load_pyramid_environment(global_config, settings)
156
157
157 includeme_first(config)
158 includeme_first(config)
158 includeme(config)
159 includeme(config)
159 pyramid_app = config.make_wsgi_app()
160 pyramid_app = config.make_wsgi_app()
160 pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
161 pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
161 pyramid_app.config = config
162 pyramid_app.config = config
163
164 # creating the app uses a connection - return it after we are done
165 meta.Session.remove()
166
162 return pyramid_app
167 return pyramid_app
163
168
164
169
165 def make_not_found_view(config):
170 def make_not_found_view(config):
166 """
171 """
167 This creates the view which should be registered as not-found-view to
172 This creates the view which should be registered as not-found-view to
168 pyramid. Basically it contains of the old pylons app, converted to a view.
173 pyramid. Basically it contains of the old pylons app, converted to a view.
169 Additionally it is wrapped by some other middlewares.
174 Additionally it is wrapped by some other middlewares.
170 """
175 """
171 settings = config.registry.settings
176 settings = config.registry.settings
172 vcs_server_enabled = settings['vcs.server.enable']
177 vcs_server_enabled = settings['vcs.server.enable']
173
178
174 # Make pylons app from unprepared settings.
179 # Make pylons app from unprepared settings.
175 pylons_app = make_app(
180 pylons_app = make_app(
176 config.registry._pylons_compat_global_config,
181 config.registry._pylons_compat_global_config,
177 **config.registry._pylons_compat_settings)
182 **config.registry._pylons_compat_settings)
178 config.registry._pylons_compat_config = pylons_app.config
183 config.registry._pylons_compat_config = pylons_app.config
179
184
180 # Appenlight monitoring.
185 # Appenlight monitoring.
181 pylons_app, appenlight_client = wrap_in_appenlight_if_enabled(
186 pylons_app, appenlight_client = wrap_in_appenlight_if_enabled(
182 pylons_app, settings)
187 pylons_app, settings)
183
188
184 # The VCSMiddleware shall operate like a fallback if pyramid doesn't find
189 # The VCSMiddleware shall operate like a fallback if pyramid doesn't find
185 # a view to handle the request. Therefore we wrap it around the pylons app.
190 # a view to handle the request. Therefore we wrap it around the pylons app.
186 if vcs_server_enabled:
191 if vcs_server_enabled:
187 pylons_app = VCSMiddleware(
192 pylons_app = VCSMiddleware(
188 pylons_app, settings, appenlight_client, registry=config.registry)
193 pylons_app, settings, appenlight_client, registry=config.registry)
189
194
190 pylons_app_as_view = wsgiapp(pylons_app)
195 pylons_app_as_view = wsgiapp(pylons_app)
191
196
192 def pylons_app_with_error_handler(context, request):
197 def pylons_app_with_error_handler(context, request):
193 """
198 """
194 Handle exceptions from rc pylons app:
199 Handle exceptions from rc pylons app:
195
200
196 - old webob type exceptions get converted to pyramid exceptions
201 - old webob type exceptions get converted to pyramid exceptions
197 - pyramid exceptions are passed to the error handler view
202 - pyramid exceptions are passed to the error handler view
198 """
203 """
199 def is_vcs_response(response):
204 def is_vcs_response(response):
200 return 'X-RhodeCode-Backend' in response.headers
205 return 'X-RhodeCode-Backend' in response.headers
201
206
202 def is_http_error(response):
207 def is_http_error(response):
203 # webob type error responses
208 # webob type error responses
204 return (400 <= response.status_int <= 599)
209 return (400 <= response.status_int <= 599)
205
210
206 def is_error_handling_needed(response):
211 def is_error_handling_needed(response):
207 return is_http_error(response) and not is_vcs_response(response)
212 return is_http_error(response) and not is_vcs_response(response)
208
213
209 try:
214 try:
210 response = pylons_app_as_view(context, request)
215 response = pylons_app_as_view(context, request)
211 if is_error_handling_needed(response):
216 if is_error_handling_needed(response):
212 response = webob_to_pyramid_http_response(response)
217 response = webob_to_pyramid_http_response(response)
213 return error_handler(response, request)
218 return error_handler(response, request)
214 except HTTPError as e: # pyramid type exceptions
219 except HTTPError as e: # pyramid type exceptions
215 return error_handler(e, request)
220 return error_handler(e, request)
216 except Exception as e:
221 except Exception as e:
217 log.exception(e)
222 log.exception(e)
218
223
219 if settings.get('debugtoolbar.enabled', False):
224 if settings.get('debugtoolbar.enabled', False):
220 raise
225 raise
221
226
222 if isinstance(e, VCSCommunicationError):
227 if isinstance(e, VCSCommunicationError):
223 return error_handler(VCSServerUnavailable(), request)
228 return error_handler(VCSServerUnavailable(), request)
224
229
225 return error_handler(HTTPInternalServerError(), request)
230 return error_handler(HTTPInternalServerError(), request)
226
231
227 return response
232 return response
228
233
229 return pylons_app_with_error_handler
234 return pylons_app_with_error_handler
230
235
231
236
232 def add_pylons_compat_data(registry, global_config, settings):
237 def add_pylons_compat_data(registry, global_config, settings):
233 """
238 """
234 Attach data to the registry to support the Pylons integration.
239 Attach data to the registry to support the Pylons integration.
235 """
240 """
236 registry._pylons_compat_global_config = global_config
241 registry._pylons_compat_global_config = global_config
237 registry._pylons_compat_settings = settings
242 registry._pylons_compat_settings = settings
238
243
239
244
240 def webob_to_pyramid_http_response(webob_response):
245 def webob_to_pyramid_http_response(webob_response):
241 ResponseClass = httpexceptions.status_map[webob_response.status_int]
246 ResponseClass = httpexceptions.status_map[webob_response.status_int]
242 pyramid_response = ResponseClass(webob_response.status)
247 pyramid_response = ResponseClass(webob_response.status)
243 pyramid_response.status = webob_response.status
248 pyramid_response.status = webob_response.status
244 pyramid_response.headers.update(webob_response.headers)
249 pyramid_response.headers.update(webob_response.headers)
245 if pyramid_response.headers['content-type'] == 'text/html':
250 if pyramid_response.headers['content-type'] == 'text/html':
246 pyramid_response.headers['content-type'] = 'text/html; charset=UTF-8'
251 pyramid_response.headers['content-type'] = 'text/html; charset=UTF-8'
247 return pyramid_response
252 return pyramid_response
248
253
249
254
250 def error_handler(exception, request):
255 def error_handler(exception, request):
251 from rhodecode.model.settings import SettingsModel
256 from rhodecode.model.settings import SettingsModel
252 from rhodecode.lib.utils2 import AttributeDict
257 from rhodecode.lib.utils2 import AttributeDict
253
258
254 try:
259 try:
255 rc_config = SettingsModel().get_all_settings()
260 rc_config = SettingsModel().get_all_settings()
256 except Exception:
261 except Exception:
257 log.exception('failed to fetch settings')
262 log.exception('failed to fetch settings')
258 rc_config = {}
263 rc_config = {}
259
264
260 base_response = HTTPInternalServerError()
265 base_response = HTTPInternalServerError()
261 # prefer original exception for the response since it may have headers set
266 # prefer original exception for the response since it may have headers set
262 if isinstance(exception, HTTPError):
267 if isinstance(exception, HTTPError):
263 base_response = exception
268 base_response = exception
264
269
265 c = AttributeDict()
270 c = AttributeDict()
266 c.error_message = base_response.status
271 c.error_message = base_response.status
267 c.error_explanation = base_response.explanation or str(base_response)
272 c.error_explanation = base_response.explanation or str(base_response)
268 c.visual = AttributeDict()
273 c.visual = AttributeDict()
269
274
270 c.visual.rhodecode_support_url = (
275 c.visual.rhodecode_support_url = (
271 request.registry.settings.get('rhodecode_support_url') or
276 request.registry.settings.get('rhodecode_support_url') or
272 request.route_url('rhodecode_support')
277 request.route_url('rhodecode_support')
273 )
278 )
274 c.redirect_time = 0
279 c.redirect_time = 0
275 c.rhodecode_name = rc_config.get('rhodecode_title', '')
280 c.rhodecode_name = rc_config.get('rhodecode_title', '')
276 if not c.rhodecode_name:
281 if not c.rhodecode_name:
277 c.rhodecode_name = 'Rhodecode'
282 c.rhodecode_name = 'Rhodecode'
278
283
279 c.causes = []
284 c.causes = []
280 if hasattr(base_response, 'causes'):
285 if hasattr(base_response, 'causes'):
281 c.causes = base_response.causes
286 c.causes = base_response.causes
282
287
283 response = render_to_response(
288 response = render_to_response(
284 '/errors/error_document.html', {'c': c}, request=request,
289 '/errors/error_document.html', {'c': c}, request=request,
285 response=base_response)
290 response=base_response)
286
291
287 return response
292 return response
288
293
289
294
290 def includeme(config):
295 def includeme(config):
291 settings = config.registry.settings
296 settings = config.registry.settings
292
297
293 # plugin information
298 # plugin information
294 config.registry.rhodecode_plugins = OrderedDict()
299 config.registry.rhodecode_plugins = OrderedDict()
295
300
296 config.add_directive(
301 config.add_directive(
297 'register_rhodecode_plugin', register_rhodecode_plugin)
302 'register_rhodecode_plugin', register_rhodecode_plugin)
298
303
299 if asbool(settings.get('appenlight', 'false')):
304 if asbool(settings.get('appenlight', 'false')):
300 config.include('appenlight_client.ext.pyramid_tween')
305 config.include('appenlight_client.ext.pyramid_tween')
301
306
302 # Includes which are required. The application would fail without them.
307 # Includes which are required. The application would fail without them.
303 config.include('pyramid_mako')
308 config.include('pyramid_mako')
304 config.include('pyramid_beaker')
309 config.include('pyramid_beaker')
305 config.include('rhodecode.channelstream')
310 config.include('rhodecode.channelstream')
306 config.include('rhodecode.admin')
311 config.include('rhodecode.admin')
307 config.include('rhodecode.authentication')
312 config.include('rhodecode.authentication')
308 config.include('rhodecode.integrations')
313 config.include('rhodecode.integrations')
309 config.include('rhodecode.login')
314 config.include('rhodecode.login')
310 config.include('rhodecode.tweens')
315 config.include('rhodecode.tweens')
311 config.include('rhodecode.api')
316 config.include('rhodecode.api')
312 config.include('rhodecode.svn_support')
317 config.include('rhodecode.svn_support')
313 config.add_route(
318 config.add_route(
314 'rhodecode_support', 'https://rhodecode.com/help/', static=True)
319 'rhodecode_support', 'https://rhodecode.com/help/', static=True)
315
320
316 # Add subscribers.
321 # Add subscribers.
317 config.add_subscriber(scan_repositories_if_enabled, ApplicationCreated)
322 config.add_subscriber(scan_repositories_if_enabled, ApplicationCreated)
318
323
319 # Set the authorization policy.
324 # Set the authorization policy.
320 authz_policy = ACLAuthorizationPolicy()
325 authz_policy = ACLAuthorizationPolicy()
321 config.set_authorization_policy(authz_policy)
326 config.set_authorization_policy(authz_policy)
322
327
323 # Set the default renderer for HTML templates to mako.
328 # Set the default renderer for HTML templates to mako.
324 config.add_mako_renderer('.html')
329 config.add_mako_renderer('.html')
325
330
326 # include RhodeCode plugins
331 # include RhodeCode plugins
327 includes = aslist(settings.get('rhodecode.includes', []))
332 includes = aslist(settings.get('rhodecode.includes', []))
328 for inc in includes:
333 for inc in includes:
329 config.include(inc)
334 config.include(inc)
330
335
331 # This is the glue which allows us to migrate in chunks. By registering the
336 # This is the glue which allows us to migrate in chunks. By registering the
332 # pylons based application as the "Not Found" view in Pyramid, we will
337 # pylons based application as the "Not Found" view in Pyramid, we will
333 # fallback to the old application each time the new one does not yet know
338 # fallback to the old application each time the new one does not yet know
334 # how to handle a request.
339 # how to handle a request.
335 config.add_notfound_view(make_not_found_view(config))
340 config.add_notfound_view(make_not_found_view(config))
336
341
337 if not settings.get('debugtoolbar.enabled', False):
342 if not settings.get('debugtoolbar.enabled', False):
338 # if no toolbar, then any exception gets caught and rendered
343 # if no toolbar, then any exception gets caught and rendered
339 config.add_view(error_handler, context=Exception)
344 config.add_view(error_handler, context=Exception)
340
345
341 config.add_view(error_handler, context=HTTPError)
346 config.add_view(error_handler, context=HTTPError)
342
347
343
348
344 def includeme_first(config):
349 def includeme_first(config):
345 # redirect automatic browser favicon.ico requests to correct place
350 # redirect automatic browser favicon.ico requests to correct place
346 def favicon_redirect(context, request):
351 def favicon_redirect(context, request):
347 return HTTPFound(
352 return HTTPFound(
348 request.static_path('rhodecode:public/images/favicon.ico'))
353 request.static_path('rhodecode:public/images/favicon.ico'))
349
354
350 config.add_view(favicon_redirect, route_name='favicon')
355 config.add_view(favicon_redirect, route_name='favicon')
351 config.add_route('favicon', '/favicon.ico')
356 config.add_route('favicon', '/favicon.ico')
352
357
353 config.add_static_view(
358 config.add_static_view(
354 '_static/deform', 'deform:static')
359 '_static/deform', 'deform:static')
355 config.add_static_view(
360 config.add_static_view(
356 '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
361 '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
357
362
358
363
359 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
364 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
360 """
365 """
361 Apply outer WSGI middlewares around the application.
366 Apply outer WSGI middlewares around the application.
362
367
363 Part of this has been moved up from the Pylons layer, so that the
368 Part of this has been moved up from the Pylons layer, so that the
364 data is also available if old Pylons code is hit through an already ported
369 data is also available if old Pylons code is hit through an already ported
365 view.
370 view.
366 """
371 """
367 settings = config.registry.settings
372 settings = config.registry.settings
368
373
369 # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
374 # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
370 pyramid_app = HttpsFixup(pyramid_app, settings)
375 pyramid_app = HttpsFixup(pyramid_app, settings)
371
376
372 # Add RoutesMiddleware to support the pylons compatibility tween during
377 # Add RoutesMiddleware to support the pylons compatibility tween during
373 # migration to pyramid.
378 # migration to pyramid.
374 pyramid_app = SkippableRoutesMiddleware(
379 pyramid_app = SkippableRoutesMiddleware(
375 pyramid_app, config.registry._pylons_compat_config['routes.map'],
380 pyramid_app, config.registry._pylons_compat_config['routes.map'],
376 skip_prefixes=(STATIC_FILE_PREFIX, '/_debug_toolbar'))
381 skip_prefixes=(STATIC_FILE_PREFIX, '/_debug_toolbar'))
377
382
378 pyramid_app, _ = wrap_in_appenlight_if_enabled(pyramid_app, settings)
383 pyramid_app, _ = wrap_in_appenlight_if_enabled(pyramid_app, settings)
379
384
380 if settings['gzip_responses']:
385 if settings['gzip_responses']:
381 pyramid_app = make_gzip_middleware(
386 pyramid_app = make_gzip_middleware(
382 pyramid_app, settings, compress_level=1)
387 pyramid_app, settings, compress_level=1)
383
388
384 return pyramid_app
389
390 # this should be the outer most middleware in the wsgi stack since
391 # middleware like Routes make database calls
392 def pyramid_app_with_cleanup(environ, start_response):
393 try:
394 return pyramid_app(environ, start_response)
395 finally:
396 # Dispose current database session and rollback uncommitted
397 # transactions.
398 meta.Session.remove()
399
400 # In a single threaded mode server, on non sqlite db we should have
401 # '0 Current Checked out connections' at the end of a request,
402 # if not, then something, somewhere is leaving a connection open
403 pool = meta.Base.metadata.bind.engine.pool
404 log.debug('sa pool status: %s', pool.status())
405
406
407 return pyramid_app_with_cleanup
385
408
386
409
387 def sanitize_settings_and_apply_defaults(settings):
410 def sanitize_settings_and_apply_defaults(settings):
388 """
411 """
389 Applies settings defaults and does all type conversion.
412 Applies settings defaults and does all type conversion.
390
413
391 We would move all settings parsing and preparation into this place, so that
414 We would move all settings parsing and preparation into this place, so that
392 we have only one place left which deals with this part. The remaining parts
415 we have only one place left which deals with this part. The remaining parts
393 of the application would start to rely fully on well prepared settings.
416 of the application would start to rely fully on well prepared settings.
394
417
395 This piece would later be split up per topic to avoid a big fat monster
418 This piece would later be split up per topic to avoid a big fat monster
396 function.
419 function.
397 """
420 """
398
421
399 # Pyramid's mako renderer has to search in the templates folder so that the
422 # Pyramid's mako renderer has to search in the templates folder so that the
400 # old templates still work. Ported and new templates are expected to use
423 # old templates still work. Ported and new templates are expected to use
401 # real asset specifications for the includes.
424 # real asset specifications for the includes.
402 mako_directories = settings.setdefault('mako.directories', [
425 mako_directories = settings.setdefault('mako.directories', [
403 # Base templates of the original Pylons application
426 # Base templates of the original Pylons application
404 'rhodecode:templates',
427 'rhodecode:templates',
405 ])
428 ])
406 log.debug(
429 log.debug(
407 "Using the following Mako template directories: %s",
430 "Using the following Mako template directories: %s",
408 mako_directories)
431 mako_directories)
409
432
410 # Default includes, possible to change as a user
433 # Default includes, possible to change as a user
411 pyramid_includes = settings.setdefault('pyramid.includes', [
434 pyramid_includes = settings.setdefault('pyramid.includes', [
412 'rhodecode.lib.middleware.request_wrapper',
435 'rhodecode.lib.middleware.request_wrapper',
413 ])
436 ])
414 log.debug(
437 log.debug(
415 "Using the following pyramid.includes: %s",
438 "Using the following pyramid.includes: %s",
416 pyramid_includes)
439 pyramid_includes)
417
440
418 # TODO: johbo: Re-think this, usually the call to config.include
441 # TODO: johbo: Re-think this, usually the call to config.include
419 # should allow to pass in a prefix.
442 # should allow to pass in a prefix.
420 settings.setdefault('rhodecode.api.url', '/_admin/api')
443 settings.setdefault('rhodecode.api.url', '/_admin/api')
421
444
422 # Sanitize generic settings.
445 # Sanitize generic settings.
423 _list_setting(settings, 'default_encoding', 'UTF-8')
446 _list_setting(settings, 'default_encoding', 'UTF-8')
424 _bool_setting(settings, 'is_test', 'false')
447 _bool_setting(settings, 'is_test', 'false')
425 _bool_setting(settings, 'gzip_responses', 'false')
448 _bool_setting(settings, 'gzip_responses', 'false')
426
449
427 # Call split out functions that sanitize settings for each topic.
450 # Call split out functions that sanitize settings for each topic.
428 _sanitize_appenlight_settings(settings)
451 _sanitize_appenlight_settings(settings)
429 _sanitize_vcs_settings(settings)
452 _sanitize_vcs_settings(settings)
430
453
431 return settings
454 return settings
432
455
433
456
434 def _sanitize_appenlight_settings(settings):
457 def _sanitize_appenlight_settings(settings):
435 _bool_setting(settings, 'appenlight', 'false')
458 _bool_setting(settings, 'appenlight', 'false')
436
459
437
460
438 def _sanitize_vcs_settings(settings):
461 def _sanitize_vcs_settings(settings):
439 """
462 """
440 Applies settings defaults and does type conversion for all VCS related
463 Applies settings defaults and does type conversion for all VCS related
441 settings.
464 settings.
442 """
465 """
443 _string_setting(settings, 'vcs.svn.compatible_version', '')
466 _string_setting(settings, 'vcs.svn.compatible_version', '')
444 _string_setting(settings, 'git_rev_filter', '--all')
467 _string_setting(settings, 'git_rev_filter', '--all')
445 _string_setting(settings, 'vcs.hooks.protocol', 'pyro4')
468 _string_setting(settings, 'vcs.hooks.protocol', 'pyro4')
446 _string_setting(settings, 'vcs.server', '')
469 _string_setting(settings, 'vcs.server', '')
447 _string_setting(settings, 'vcs.server.log_level', 'debug')
470 _string_setting(settings, 'vcs.server.log_level', 'debug')
448 _string_setting(settings, 'vcs.server.protocol', 'pyro4')
471 _string_setting(settings, 'vcs.server.protocol', 'pyro4')
449 _bool_setting(settings, 'startup.import_repos', 'false')
472 _bool_setting(settings, 'startup.import_repos', 'false')
450 _bool_setting(settings, 'vcs.hooks.direct_calls', 'false')
473 _bool_setting(settings, 'vcs.hooks.direct_calls', 'false')
451 _bool_setting(settings, 'vcs.server.enable', 'true')
474 _bool_setting(settings, 'vcs.server.enable', 'true')
452 _bool_setting(settings, 'vcs.start_server', 'false')
475 _bool_setting(settings, 'vcs.start_server', 'false')
453 _list_setting(settings, 'vcs.backends', 'hg, git, svn')
476 _list_setting(settings, 'vcs.backends', 'hg, git, svn')
454 _int_setting(settings, 'vcs.connection_timeout', 3600)
477 _int_setting(settings, 'vcs.connection_timeout', 3600)
455
478
456
479
457 def _int_setting(settings, name, default):
480 def _int_setting(settings, name, default):
458 settings[name] = int(settings.get(name, default))
481 settings[name] = int(settings.get(name, default))
459
482
460
483
461 def _bool_setting(settings, name, default):
484 def _bool_setting(settings, name, default):
462 input = settings.get(name, default)
485 input = settings.get(name, default)
463 if isinstance(input, unicode):
486 if isinstance(input, unicode):
464 input = input.encode('utf8')
487 input = input.encode('utf8')
465 settings[name] = asbool(input)
488 settings[name] = asbool(input)
466
489
467
490
468 def _list_setting(settings, name, default):
491 def _list_setting(settings, name, default):
469 raw_value = settings.get(name, default)
492 raw_value = settings.get(name, default)
470
493
471 old_separator = ','
494 old_separator = ','
472 if old_separator in raw_value:
495 if old_separator in raw_value:
473 # If we get a comma separated list, pass it to our own function.
496 # If we get a comma separated list, pass it to our own function.
474 settings[name] = rhodecode_aslist(raw_value, sep=old_separator)
497 settings[name] = rhodecode_aslist(raw_value, sep=old_separator)
475 else:
498 else:
476 # Otherwise we assume it uses pyramids space/newline separation.
499 # Otherwise we assume it uses pyramids space/newline separation.
477 settings[name] = aslist(raw_value)
500 settings[name] = aslist(raw_value)
478
501
479
502
480 def _string_setting(settings, name, default):
503 def _string_setting(settings, name, default):
481 settings[name] = settings.get(name, default).lower()
504 settings[name] = settings.get(name, default).lower()
@@ -1,874 +1,885 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2011-2016 RhodeCode GmbH
3 # Copyright (C) 2011-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21
21
22 """
22 """
23 Set of diffing helpers, previously part of vcs
23 Set of diffing helpers, previously part of vcs
24 """
24 """
25
25
26 import collections
26 import collections
27 import re
27 import re
28 import difflib
28 import difflib
29 import logging
29 import logging
30
30
31 from itertools import tee, imap
31 from itertools import tee, imap
32
32
33 from pylons.i18n.translation import _
33 from pylons.i18n.translation import _
34
34
35 from rhodecode.lib.vcs.exceptions import VCSError
35 from rhodecode.lib.vcs.exceptions import VCSError
36 from rhodecode.lib.vcs.nodes import FileNode, SubModuleNode
36 from rhodecode.lib.vcs.nodes import FileNode, SubModuleNode
37 from rhodecode.lib.vcs.backends.base import EmptyCommit
37 from rhodecode.lib.vcs.backends.base import EmptyCommit
38 from rhodecode.lib.helpers import escape
38 from rhodecode.lib.helpers import escape
39 from rhodecode.lib.utils2 import safe_unicode
39 from rhodecode.lib.utils2 import safe_unicode
40
40
41 log = logging.getLogger(__name__)
41 log = logging.getLogger(__name__)
42
42
43 # define max context, a file with more than this numbers of lines is unusable
44 # in browser anyway
45 MAX_CONTEXT = 1024 * 1014
46
43
47
44 class OPS(object):
48 class OPS(object):
45 ADD = 'A'
49 ADD = 'A'
46 MOD = 'M'
50 MOD = 'M'
47 DEL = 'D'
51 DEL = 'D'
48
52
53
49 def wrap_to_table(str_):
54 def wrap_to_table(str_):
50 return '''<table class="code-difftable">
55 return '''<table class="code-difftable">
51 <tr class="line no-comment">
56 <tr class="line no-comment">
52 <td class="add-comment-line tooltip" title="%s"><span class="add-comment-content"></span></td>
57 <td class="add-comment-line tooltip" title="%s"><span class="add-comment-content"></span></td>
53 <td class="lineno new"></td>
58 <td class="lineno new"></td>
54 <td class="code no-comment"><pre>%s</pre></td>
59 <td class="code no-comment"><pre>%s</pre></td>
55 </tr>
60 </tr>
56 </table>''' % (_('Click to comment'), str_)
61 </table>''' % (_('Click to comment'), str_)
57
62
58
63
59 def wrapped_diff(filenode_old, filenode_new, diff_limit=None, file_limit=None,
64 def wrapped_diff(filenode_old, filenode_new, diff_limit=None, file_limit=None,
60 show_full_diff=False, ignore_whitespace=True, line_context=3,
65 show_full_diff=False, ignore_whitespace=True, line_context=3,
61 enable_comments=False):
66 enable_comments=False):
62 """
67 """
63 returns a wrapped diff into a table, checks for cut_off_limit for file and
68 returns a wrapped diff into a table, checks for cut_off_limit for file and
64 whole diff and presents proper message
69 whole diff and presents proper message
65 """
70 """
66
71
67 if filenode_old is None:
72 if filenode_old is None:
68 filenode_old = FileNode(filenode_new.path, '', EmptyCommit())
73 filenode_old = FileNode(filenode_new.path, '', EmptyCommit())
69
74
70 if filenode_old.is_binary or filenode_new.is_binary:
75 if filenode_old.is_binary or filenode_new.is_binary:
71 diff = wrap_to_table(_('Binary file'))
76 diff = wrap_to_table(_('Binary file'))
72 stats = None
77 stats = None
73 size = 0
78 size = 0
74 data = None
79 data = None
75
80
76 elif diff_limit != -1 and (diff_limit is None or
81 elif diff_limit != -1 and (diff_limit is None or
77 (filenode_old.size < diff_limit and filenode_new.size < diff_limit)):
82 (filenode_old.size < diff_limit and filenode_new.size < diff_limit)):
78
83
79 f_gitdiff = get_gitdiff(filenode_old, filenode_new,
84 f_gitdiff = get_gitdiff(filenode_old, filenode_new,
80 ignore_whitespace=ignore_whitespace,
85 ignore_whitespace=ignore_whitespace,
81 context=line_context)
86 context=line_context)
82 diff_processor = DiffProcessor(f_gitdiff, format='gitdiff', diff_limit=diff_limit,
87 diff_processor = DiffProcessor(
88 f_gitdiff, format='gitdiff', diff_limit=diff_limit,
83 file_limit=file_limit, show_full_diff=show_full_diff)
89 file_limit=file_limit, show_full_diff=show_full_diff)
84 _parsed = diff_processor.prepare()
90 _parsed = diff_processor.prepare()
85
91
86 diff = diff_processor.as_html(enable_comments=enable_comments)
92 diff = diff_processor.as_html(enable_comments=enable_comments)
87 stats = _parsed[0]['stats'] if _parsed else None
93 stats = _parsed[0]['stats'] if _parsed else None
88 size = len(diff or '')
94 size = len(diff or '')
89 data = _parsed[0] if _parsed else None
95 data = _parsed[0] if _parsed else None
90 else:
96 else:
91 diff = wrap_to_table(_('Changeset was too big and was cut off, use '
97 diff = wrap_to_table(_('Changeset was too big and was cut off, use '
92 'diff menu to display this diff'))
98 'diff menu to display this diff'))
93 stats = None
99 stats = None
94 size = 0
100 size = 0
95 data = None
101 data = None
96 if not diff:
102 if not diff:
97 submodules = filter(lambda o: isinstance(o, SubModuleNode),
103 submodules = filter(lambda o: isinstance(o, SubModuleNode),
98 [filenode_new, filenode_old])
104 [filenode_new, filenode_old])
99 if submodules:
105 if submodules:
100 diff = wrap_to_table(escape('Submodule %r' % submodules[0]))
106 diff = wrap_to_table(escape('Submodule %r' % submodules[0]))
101 else:
107 else:
102 diff = wrap_to_table(_('No changes detected'))
108 diff = wrap_to_table(_('No changes detected'))
103
109
104 cs1 = filenode_old.commit.raw_id
110 cs1 = filenode_old.commit.raw_id
105 cs2 = filenode_new.commit.raw_id
111 cs2 = filenode_new.commit.raw_id
106
112
107 return size, cs1, cs2, diff, stats, data
113 return size, cs1, cs2, diff, stats, data
108
114
109
115
110 def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True, context=3):
116 def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True, context=3):
111 """
117 """
112 Returns git style diff between given ``filenode_old`` and ``filenode_new``.
118 Returns git style diff between given ``filenode_old`` and ``filenode_new``.
113
119
114 :param ignore_whitespace: ignore whitespaces in diff
120 :param ignore_whitespace: ignore whitespaces in diff
115 """
121 """
116 # make sure we pass in default context
122 # make sure we pass in default context
117 context = context or 3
123 context = context or 3
124 # protect against IntOverflow when passing HUGE context
125 if context > MAX_CONTEXT:
126 context = MAX_CONTEXT
127
118 submodules = filter(lambda o: isinstance(o, SubModuleNode),
128 submodules = filter(lambda o: isinstance(o, SubModuleNode),
119 [filenode_new, filenode_old])
129 [filenode_new, filenode_old])
120 if submodules:
130 if submodules:
121 return ''
131 return ''
122
132
123 for filenode in (filenode_old, filenode_new):
133 for filenode in (filenode_old, filenode_new):
124 if not isinstance(filenode, FileNode):
134 if not isinstance(filenode, FileNode):
125 raise VCSError(
135 raise VCSError(
126 "Given object should be FileNode object, not %s"
136 "Given object should be FileNode object, not %s"
127 % filenode.__class__)
137 % filenode.__class__)
128
138
129 repo = filenode_new.commit.repository
139 repo = filenode_new.commit.repository
130 old_commit = filenode_old.commit or repo.EMPTY_COMMIT
140 old_commit = filenode_old.commit or repo.EMPTY_COMMIT
131 new_commit = filenode_new.commit
141 new_commit = filenode_new.commit
132
142
133 vcs_gitdiff = repo.get_diff(
143 vcs_gitdiff = repo.get_diff(
134 old_commit, new_commit, filenode_new.path,
144 old_commit, new_commit, filenode_new.path,
135 ignore_whitespace, context, path1=filenode_old.path)
145 ignore_whitespace, context, path1=filenode_old.path)
136 return vcs_gitdiff
146 return vcs_gitdiff
137
147
138 NEW_FILENODE = 1
148 NEW_FILENODE = 1
139 DEL_FILENODE = 2
149 DEL_FILENODE = 2
140 MOD_FILENODE = 3
150 MOD_FILENODE = 3
141 RENAMED_FILENODE = 4
151 RENAMED_FILENODE = 4
142 COPIED_FILENODE = 5
152 COPIED_FILENODE = 5
143 CHMOD_FILENODE = 6
153 CHMOD_FILENODE = 6
144 BIN_FILENODE = 7
154 BIN_FILENODE = 7
145
155
146
156
147 class LimitedDiffContainer(object):
157 class LimitedDiffContainer(object):
148
158
149 def __init__(self, diff_limit, cur_diff_size, diff):
159 def __init__(self, diff_limit, cur_diff_size, diff):
150 self.diff = diff
160 self.diff = diff
151 self.diff_limit = diff_limit
161 self.diff_limit = diff_limit
152 self.cur_diff_size = cur_diff_size
162 self.cur_diff_size = cur_diff_size
153
163
154 def __getitem__(self, key):
164 def __getitem__(self, key):
155 return self.diff.__getitem__(key)
165 return self.diff.__getitem__(key)
156
166
157 def __iter__(self):
167 def __iter__(self):
158 for l in self.diff:
168 for l in self.diff:
159 yield l
169 yield l
160
170
161
171
162 class Action(object):
172 class Action(object):
163 """
173 """
164 Contains constants for the action value of the lines in a parsed diff.
174 Contains constants for the action value of the lines in a parsed diff.
165 """
175 """
166
176
167 ADD = 'add'
177 ADD = 'add'
168 DELETE = 'del'
178 DELETE = 'del'
169 UNMODIFIED = 'unmod'
179 UNMODIFIED = 'unmod'
170
180
171 CONTEXT = 'context'
181 CONTEXT = 'context'
172
182
173
183
174 class DiffProcessor(object):
184 class DiffProcessor(object):
175 """
185 """
176 Give it a unified or git diff and it returns a list of the files that were
186 Give it a unified or git diff and it returns a list of the files that were
177 mentioned in the diff together with a dict of meta information that
187 mentioned in the diff together with a dict of meta information that
178 can be used to render it in a HTML template.
188 can be used to render it in a HTML template.
179
189
180 .. note:: Unicode handling
190 .. note:: Unicode handling
181
191
182 The original diffs are a byte sequence and can contain filenames
192 The original diffs are a byte sequence and can contain filenames
183 in mixed encodings. This class generally returns `unicode` objects
193 in mixed encodings. This class generally returns `unicode` objects
184 since the result is intended for presentation to the user.
194 since the result is intended for presentation to the user.
185
195
186 """
196 """
187 _chunk_re = re.compile(r'^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')
197 _chunk_re = re.compile(r'^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')
188 _newline_marker = re.compile(r'^\\ No newline at end of file')
198 _newline_marker = re.compile(r'^\\ No newline at end of file')
189
199
190 # used for inline highlighter word split
200 # used for inline highlighter word split
191 _token_re = re.compile(r'()(&gt;|&lt;|&amp;|\W+?)')
201 _token_re = re.compile(r'()(&gt;|&lt;|&amp;|\W+?)')
192
202
193 def __init__(self, diff, format='gitdiff', diff_limit=None, file_limit=None, show_full_diff=True):
203 def __init__(self, diff, format='gitdiff', diff_limit=None,
204 file_limit=None, show_full_diff=True):
194 """
205 """
195 :param diff: A `Diff` object representing a diff from a vcs backend
206 :param diff: A `Diff` object representing a diff from a vcs backend
196 :param format: format of diff passed, `udiff` or `gitdiff`
207 :param format: format of diff passed, `udiff` or `gitdiff`
197 :param diff_limit: define the size of diff that is considered "big"
208 :param diff_limit: define the size of diff that is considered "big"
198 based on that parameter cut off will be triggered, set to None
209 based on that parameter cut off will be triggered, set to None
199 to show full diff
210 to show full diff
200 """
211 """
201 self._diff = diff
212 self._diff = diff
202 self._format = format
213 self._format = format
203 self.adds = 0
214 self.adds = 0
204 self.removes = 0
215 self.removes = 0
205 # calculate diff size
216 # calculate diff size
206 self.diff_limit = diff_limit
217 self.diff_limit = diff_limit
207 self.file_limit = file_limit
218 self.file_limit = file_limit
208 self.show_full_diff = show_full_diff
219 self.show_full_diff = show_full_diff
209 self.cur_diff_size = 0
220 self.cur_diff_size = 0
210 self.parsed = False
221 self.parsed = False
211 self.parsed_diff = []
222 self.parsed_diff = []
212
223
213 if format == 'gitdiff':
224 if format == 'gitdiff':
214 self.differ = self._highlight_line_difflib
225 self.differ = self._highlight_line_difflib
215 self._parser = self._parse_gitdiff
226 self._parser = self._parse_gitdiff
216 else:
227 else:
217 self.differ = self._highlight_line_udiff
228 self.differ = self._highlight_line_udiff
218 self._parser = self._parse_udiff
229 self._parser = self._parse_udiff
219
230
220 def _copy_iterator(self):
231 def _copy_iterator(self):
221 """
232 """
222 make a fresh copy of generator, we should not iterate thru
233 make a fresh copy of generator, we should not iterate thru
223 an original as it's needed for repeating operations on
234 an original as it's needed for repeating operations on
224 this instance of DiffProcessor
235 this instance of DiffProcessor
225 """
236 """
226 self.__udiff, iterator_copy = tee(self.__udiff)
237 self.__udiff, iterator_copy = tee(self.__udiff)
227 return iterator_copy
238 return iterator_copy
228
239
229 def _escaper(self, string):
240 def _escaper(self, string):
230 """
241 """
231 Escaper for diff escapes special chars and checks the diff limit
242 Escaper for diff escapes special chars and checks the diff limit
232
243
233 :param string:
244 :param string:
234 """
245 """
235
246
236 self.cur_diff_size += len(string)
247 self.cur_diff_size += len(string)
237
248
238 if not self.show_full_diff and (self.cur_diff_size > self.diff_limit):
249 if not self.show_full_diff and (self.cur_diff_size > self.diff_limit):
239 raise DiffLimitExceeded('Diff Limit Exceeded')
250 raise DiffLimitExceeded('Diff Limit Exceeded')
240
251
241 return safe_unicode(string)\
252 return safe_unicode(string)\
242 .replace('&', '&amp;')\
253 .replace('&', '&amp;')\
243 .replace('<', '&lt;')\
254 .replace('<', '&lt;')\
244 .replace('>', '&gt;')
255 .replace('>', '&gt;')
245
256
246 def _line_counter(self, l):
257 def _line_counter(self, l):
247 """
258 """
248 Checks each line and bumps total adds/removes for this diff
259 Checks each line and bumps total adds/removes for this diff
249
260
250 :param l:
261 :param l:
251 """
262 """
252 if l.startswith('+') and not l.startswith('+++'):
263 if l.startswith('+') and not l.startswith('+++'):
253 self.adds += 1
264 self.adds += 1
254 elif l.startswith('-') and not l.startswith('---'):
265 elif l.startswith('-') and not l.startswith('---'):
255 self.removes += 1
266 self.removes += 1
256 return safe_unicode(l)
267 return safe_unicode(l)
257
268
258 def _highlight_line_difflib(self, line, next_):
269 def _highlight_line_difflib(self, line, next_):
259 """
270 """
260 Highlight inline changes in both lines.
271 Highlight inline changes in both lines.
261 """
272 """
262
273
263 if line['action'] == Action.DELETE:
274 if line['action'] == Action.DELETE:
264 old, new = line, next_
275 old, new = line, next_
265 else:
276 else:
266 old, new = next_, line
277 old, new = next_, line
267
278
268 oldwords = self._token_re.split(old['line'])
279 oldwords = self._token_re.split(old['line'])
269 newwords = self._token_re.split(new['line'])
280 newwords = self._token_re.split(new['line'])
270 sequence = difflib.SequenceMatcher(None, oldwords, newwords)
281 sequence = difflib.SequenceMatcher(None, oldwords, newwords)
271
282
272 oldfragments, newfragments = [], []
283 oldfragments, newfragments = [], []
273 for tag, i1, i2, j1, j2 in sequence.get_opcodes():
284 for tag, i1, i2, j1, j2 in sequence.get_opcodes():
274 oldfrag = ''.join(oldwords[i1:i2])
285 oldfrag = ''.join(oldwords[i1:i2])
275 newfrag = ''.join(newwords[j1:j2])
286 newfrag = ''.join(newwords[j1:j2])
276 if tag != 'equal':
287 if tag != 'equal':
277 if oldfrag:
288 if oldfrag:
278 oldfrag = '<del>%s</del>' % oldfrag
289 oldfrag = '<del>%s</del>' % oldfrag
279 if newfrag:
290 if newfrag:
280 newfrag = '<ins>%s</ins>' % newfrag
291 newfrag = '<ins>%s</ins>' % newfrag
281 oldfragments.append(oldfrag)
292 oldfragments.append(oldfrag)
282 newfragments.append(newfrag)
293 newfragments.append(newfrag)
283
294
284 old['line'] = "".join(oldfragments)
295 old['line'] = "".join(oldfragments)
285 new['line'] = "".join(newfragments)
296 new['line'] = "".join(newfragments)
286
297
287 def _highlight_line_udiff(self, line, next_):
298 def _highlight_line_udiff(self, line, next_):
288 """
299 """
289 Highlight inline changes in both lines.
300 Highlight inline changes in both lines.
290 """
301 """
291 start = 0
302 start = 0
292 limit = min(len(line['line']), len(next_['line']))
303 limit = min(len(line['line']), len(next_['line']))
293 while start < limit and line['line'][start] == next_['line'][start]:
304 while start < limit and line['line'][start] == next_['line'][start]:
294 start += 1
305 start += 1
295 end = -1
306 end = -1
296 limit -= start
307 limit -= start
297 while -end <= limit and line['line'][end] == next_['line'][end]:
308 while -end <= limit and line['line'][end] == next_['line'][end]:
298 end -= 1
309 end -= 1
299 end += 1
310 end += 1
300 if start or end:
311 if start or end:
301 def do(l):
312 def do(l):
302 last = end + len(l['line'])
313 last = end + len(l['line'])
303 if l['action'] == Action.ADD:
314 if l['action'] == Action.ADD:
304 tag = 'ins'
315 tag = 'ins'
305 else:
316 else:
306 tag = 'del'
317 tag = 'del'
307 l['line'] = '%s<%s>%s</%s>%s' % (
318 l['line'] = '%s<%s>%s</%s>%s' % (
308 l['line'][:start],
319 l['line'][:start],
309 tag,
320 tag,
310 l['line'][start:last],
321 l['line'][start:last],
311 tag,
322 tag,
312 l['line'][last:]
323 l['line'][last:]
313 )
324 )
314 do(line)
325 do(line)
315 do(next_)
326 do(next_)
316
327
317 def _clean_line(self, line, command):
328 def _clean_line(self, line, command):
318 if command in ['+', '-', ' ']:
329 if command in ['+', '-', ' ']:
319 # only modify the line if it's actually a diff thing
330 # only modify the line if it's actually a diff thing
320 line = line[1:]
331 line = line[1:]
321 return line
332 return line
322
333
323 def _parse_gitdiff(self, inline_diff=True):
334 def _parse_gitdiff(self, inline_diff=True):
324 _files = []
335 _files = []
325 diff_container = lambda arg: arg
336 diff_container = lambda arg: arg
326
337
327 for chunk in self._diff.chunks():
338 for chunk in self._diff.chunks():
328 head = chunk.header
339 head = chunk.header
329
340
330 diff = imap(self._escaper, chunk.diff.splitlines(1))
341 diff = imap(self._escaper, chunk.diff.splitlines(1))
331 raw_diff = chunk.raw
342 raw_diff = chunk.raw
332 limited_diff = False
343 limited_diff = False
333 exceeds_limit = False
344 exceeds_limit = False
334
345
335 op = None
346 op = None
336 stats = {
347 stats = {
337 'added': 0,
348 'added': 0,
338 'deleted': 0,
349 'deleted': 0,
339 'binary': False,
350 'binary': False,
340 'ops': {},
351 'ops': {},
341 }
352 }
342
353
343 if head['deleted_file_mode']:
354 if head['deleted_file_mode']:
344 op = OPS.DEL
355 op = OPS.DEL
345 stats['binary'] = True
356 stats['binary'] = True
346 stats['ops'][DEL_FILENODE] = 'deleted file'
357 stats['ops'][DEL_FILENODE] = 'deleted file'
347
358
348 elif head['new_file_mode']:
359 elif head['new_file_mode']:
349 op = OPS.ADD
360 op = OPS.ADD
350 stats['binary'] = True
361 stats['binary'] = True
351 stats['ops'][NEW_FILENODE] = 'new file %s' % head['new_file_mode']
362 stats['ops'][NEW_FILENODE] = 'new file %s' % head['new_file_mode']
352 else: # modify operation, can be copy, rename or chmod
363 else: # modify operation, can be copy, rename or chmod
353
364
354 # CHMOD
365 # CHMOD
355 if head['new_mode'] and head['old_mode']:
366 if head['new_mode'] and head['old_mode']:
356 op = OPS.MOD
367 op = OPS.MOD
357 stats['binary'] = True
368 stats['binary'] = True
358 stats['ops'][CHMOD_FILENODE] = (
369 stats['ops'][CHMOD_FILENODE] = (
359 'modified file chmod %s => %s' % (
370 'modified file chmod %s => %s' % (
360 head['old_mode'], head['new_mode']))
371 head['old_mode'], head['new_mode']))
361 # RENAME
372 # RENAME
362 if head['rename_from'] != head['rename_to']:
373 if head['rename_from'] != head['rename_to']:
363 op = OPS.MOD
374 op = OPS.MOD
364 stats['binary'] = True
375 stats['binary'] = True
365 stats['ops'][RENAMED_FILENODE] = (
376 stats['ops'][RENAMED_FILENODE] = (
366 'file renamed from %s to %s' % (
377 'file renamed from %s to %s' % (
367 head['rename_from'], head['rename_to']))
378 head['rename_from'], head['rename_to']))
368 # COPY
379 # COPY
369 if head.get('copy_from') and head.get('copy_to'):
380 if head.get('copy_from') and head.get('copy_to'):
370 op = OPS.MOD
381 op = OPS.MOD
371 stats['binary'] = True
382 stats['binary'] = True
372 stats['ops'][COPIED_FILENODE] = (
383 stats['ops'][COPIED_FILENODE] = (
373 'file copied from %s to %s' % (
384 'file copied from %s to %s' % (
374 head['copy_from'], head['copy_to']))
385 head['copy_from'], head['copy_to']))
375
386
376 # If our new parsed headers didn't match anything fallback to
387 # If our new parsed headers didn't match anything fallback to
377 # old style detection
388 # old style detection
378 if op is None:
389 if op is None:
379 if not head['a_file'] and head['b_file']:
390 if not head['a_file'] and head['b_file']:
380 op = OPS.ADD
391 op = OPS.ADD
381 stats['binary'] = True
392 stats['binary'] = True
382 stats['ops'][NEW_FILENODE] = 'new file'
393 stats['ops'][NEW_FILENODE] = 'new file'
383
394
384 elif head['a_file'] and not head['b_file']:
395 elif head['a_file'] and not head['b_file']:
385 op = OPS.DEL
396 op = OPS.DEL
386 stats['binary'] = True
397 stats['binary'] = True
387 stats['ops'][DEL_FILENODE] = 'deleted file'
398 stats['ops'][DEL_FILENODE] = 'deleted file'
388
399
389 # it's not ADD not DELETE
400 # it's not ADD not DELETE
390 if op is None:
401 if op is None:
391 op = OPS.MOD
402 op = OPS.MOD
392 stats['binary'] = True
403 stats['binary'] = True
393 stats['ops'][MOD_FILENODE] = 'modified file'
404 stats['ops'][MOD_FILENODE] = 'modified file'
394
405
395 # a real non-binary diff
406 # a real non-binary diff
396 if head['a_file'] or head['b_file']:
407 if head['a_file'] or head['b_file']:
397 try:
408 try:
398 raw_diff, chunks, _stats = self._parse_lines(diff)
409 raw_diff, chunks, _stats = self._parse_lines(diff)
399 stats['binary'] = False
410 stats['binary'] = False
400 stats['added'] = _stats[0]
411 stats['added'] = _stats[0]
401 stats['deleted'] = _stats[1]
412 stats['deleted'] = _stats[1]
402 # explicit mark that it's a modified file
413 # explicit mark that it's a modified file
403 if op == OPS.MOD:
414 if op == OPS.MOD:
404 stats['ops'][MOD_FILENODE] = 'modified file'
415 stats['ops'][MOD_FILENODE] = 'modified file'
405 exceeds_limit = len(raw_diff) > self.file_limit
416 exceeds_limit = len(raw_diff) > self.file_limit
406
417
407 # changed from _escaper function so we validate size of
418 # changed from _escaper function so we validate size of
408 # each file instead of the whole diff
419 # each file instead of the whole diff
409 # diff will hide big files but still show small ones
420 # diff will hide big files but still show small ones
410 # from my tests, big files are fairly safe to be parsed
421 # from my tests, big files are fairly safe to be parsed
411 # but the browser is the bottleneck
422 # but the browser is the bottleneck
412 if not self.show_full_diff and exceeds_limit:
423 if not self.show_full_diff and exceeds_limit:
413 raise DiffLimitExceeded('File Limit Exceeded')
424 raise DiffLimitExceeded('File Limit Exceeded')
414
425
415 except DiffLimitExceeded:
426 except DiffLimitExceeded:
416 diff_container = lambda _diff: \
427 diff_container = lambda _diff: \
417 LimitedDiffContainer(
428 LimitedDiffContainer(
418 self.diff_limit, self.cur_diff_size, _diff)
429 self.diff_limit, self.cur_diff_size, _diff)
419
430
420 exceeds_limit = len(raw_diff) > self.file_limit
431 exceeds_limit = len(raw_diff) > self.file_limit
421 limited_diff = True
432 limited_diff = True
422 chunks = []
433 chunks = []
423
434
424 else: # GIT format binary patch, or possibly empty diff
435 else: # GIT format binary patch, or possibly empty diff
425 if head['bin_patch']:
436 if head['bin_patch']:
426 # we have operation already extracted, but we mark simply
437 # we have operation already extracted, but we mark simply
427 # it's a diff we wont show for binary files
438 # it's a diff we wont show for binary files
428 stats['ops'][BIN_FILENODE] = 'binary diff hidden'
439 stats['ops'][BIN_FILENODE] = 'binary diff hidden'
429 chunks = []
440 chunks = []
430
441
431 if chunks and not self.show_full_diff and op == OPS.DEL:
442 if chunks and not self.show_full_diff and op == OPS.DEL:
432 # if not full diff mode show deleted file contents
443 # if not full diff mode show deleted file contents
433 # TODO: anderson: if the view is not too big, there is no way
444 # TODO: anderson: if the view is not too big, there is no way
434 # to see the content of the file
445 # to see the content of the file
435 chunks = []
446 chunks = []
436
447
437 chunks.insert(0, [{
448 chunks.insert(0, [{
438 'old_lineno': '',
449 'old_lineno': '',
439 'new_lineno': '',
450 'new_lineno': '',
440 'action': Action.CONTEXT,
451 'action': Action.CONTEXT,
441 'line': msg,
452 'line': msg,
442 } for _op, msg in stats['ops'].iteritems()
453 } for _op, msg in stats['ops'].iteritems()
443 if _op not in [MOD_FILENODE]])
454 if _op not in [MOD_FILENODE]])
444
455
445 _files.append({
456 _files.append({
446 'filename': safe_unicode(head['b_path']),
457 'filename': safe_unicode(head['b_path']),
447 'old_revision': head['a_blob_id'],
458 'old_revision': head['a_blob_id'],
448 'new_revision': head['b_blob_id'],
459 'new_revision': head['b_blob_id'],
449 'chunks': chunks,
460 'chunks': chunks,
450 'raw_diff': safe_unicode(raw_diff),
461 'raw_diff': safe_unicode(raw_diff),
451 'operation': op,
462 'operation': op,
452 'stats': stats,
463 'stats': stats,
453 'exceeds_limit': exceeds_limit,
464 'exceeds_limit': exceeds_limit,
454 'is_limited_diff': limited_diff,
465 'is_limited_diff': limited_diff,
455 })
466 })
456
467
457 sorter = lambda info: {OPS.ADD: 0, OPS.MOD: 1,
468 sorter = lambda info: {OPS.ADD: 0, OPS.MOD: 1,
458 OPS.DEL: 2}.get(info['operation'])
469 OPS.DEL: 2}.get(info['operation'])
459
470
460 if not inline_diff:
471 if not inline_diff:
461 return diff_container(sorted(_files, key=sorter))
472 return diff_container(sorted(_files, key=sorter))
462
473
463 # highlight inline changes
474 # highlight inline changes
464 for diff_data in _files:
475 for diff_data in _files:
465 for chunk in diff_data['chunks']:
476 for chunk in diff_data['chunks']:
466 lineiter = iter(chunk)
477 lineiter = iter(chunk)
467 try:
478 try:
468 while 1:
479 while 1:
469 line = lineiter.next()
480 line = lineiter.next()
470 if line['action'] not in (
481 if line['action'] not in (
471 Action.UNMODIFIED, Action.CONTEXT):
482 Action.UNMODIFIED, Action.CONTEXT):
472 nextline = lineiter.next()
483 nextline = lineiter.next()
473 if nextline['action'] in ['unmod', 'context'] or \
484 if nextline['action'] in ['unmod', 'context'] or \
474 nextline['action'] == line['action']:
485 nextline['action'] == line['action']:
475 continue
486 continue
476 self.differ(line, nextline)
487 self.differ(line, nextline)
477 except StopIteration:
488 except StopIteration:
478 pass
489 pass
479
490
480 return diff_container(sorted(_files, key=sorter))
491 return diff_container(sorted(_files, key=sorter))
481
492
482 def _parse_udiff(self, inline_diff=True):
493 def _parse_udiff(self, inline_diff=True):
483 raise NotImplementedError()
494 raise NotImplementedError()
484
495
485 def _parse_lines(self, diff):
496 def _parse_lines(self, diff):
486 """
497 """
487 Parse the diff an return data for the template.
498 Parse the diff an return data for the template.
488 """
499 """
489
500
490 lineiter = iter(diff)
501 lineiter = iter(diff)
491 stats = [0, 0]
502 stats = [0, 0]
492 chunks = []
503 chunks = []
493 raw_diff = []
504 raw_diff = []
494
505
495 try:
506 try:
496 line = lineiter.next()
507 line = lineiter.next()
497
508
498 while line:
509 while line:
499 raw_diff.append(line)
510 raw_diff.append(line)
500 lines = []
511 lines = []
501 chunks.append(lines)
512 chunks.append(lines)
502
513
503 match = self._chunk_re.match(line)
514 match = self._chunk_re.match(line)
504
515
505 if not match:
516 if not match:
506 break
517 break
507
518
508 gr = match.groups()
519 gr = match.groups()
509 (old_line, old_end,
520 (old_line, old_end,
510 new_line, new_end) = [int(x or 1) for x in gr[:-1]]
521 new_line, new_end) = [int(x or 1) for x in gr[:-1]]
511 old_line -= 1
522 old_line -= 1
512 new_line -= 1
523 new_line -= 1
513
524
514 context = len(gr) == 5
525 context = len(gr) == 5
515 old_end += old_line
526 old_end += old_line
516 new_end += new_line
527 new_end += new_line
517
528
518 if context:
529 if context:
519 # skip context only if it's first line
530 # skip context only if it's first line
520 if int(gr[0]) > 1:
531 if int(gr[0]) > 1:
521 lines.append({
532 lines.append({
522 'old_lineno': '...',
533 'old_lineno': '...',
523 'new_lineno': '...',
534 'new_lineno': '...',
524 'action': Action.CONTEXT,
535 'action': Action.CONTEXT,
525 'line': line,
536 'line': line,
526 })
537 })
527
538
528 line = lineiter.next()
539 line = lineiter.next()
529
540
530 while old_line < old_end or new_line < new_end:
541 while old_line < old_end or new_line < new_end:
531 command = ' '
542 command = ' '
532 if line:
543 if line:
533 command = line[0]
544 command = line[0]
534
545
535 affects_old = affects_new = False
546 affects_old = affects_new = False
536
547
537 # ignore those if we don't expect them
548 # ignore those if we don't expect them
538 if command in '#@':
549 if command in '#@':
539 continue
550 continue
540 elif command == '+':
551 elif command == '+':
541 affects_new = True
552 affects_new = True
542 action = Action.ADD
553 action = Action.ADD
543 stats[0] += 1
554 stats[0] += 1
544 elif command == '-':
555 elif command == '-':
545 affects_old = True
556 affects_old = True
546 action = Action.DELETE
557 action = Action.DELETE
547 stats[1] += 1
558 stats[1] += 1
548 else:
559 else:
549 affects_old = affects_new = True
560 affects_old = affects_new = True
550 action = Action.UNMODIFIED
561 action = Action.UNMODIFIED
551
562
552 if not self._newline_marker.match(line):
563 if not self._newline_marker.match(line):
553 old_line += affects_old
564 old_line += affects_old
554 new_line += affects_new
565 new_line += affects_new
555 lines.append({
566 lines.append({
556 'old_lineno': affects_old and old_line or '',
567 'old_lineno': affects_old and old_line or '',
557 'new_lineno': affects_new and new_line or '',
568 'new_lineno': affects_new and new_line or '',
558 'action': action,
569 'action': action,
559 'line': self._clean_line(line, command)
570 'line': self._clean_line(line, command)
560 })
571 })
561 raw_diff.append(line)
572 raw_diff.append(line)
562
573
563 line = lineiter.next()
574 line = lineiter.next()
564
575
565 if self._newline_marker.match(line):
576 if self._newline_marker.match(line):
566 # we need to append to lines, since this is not
577 # we need to append to lines, since this is not
567 # counted in the line specs of diff
578 # counted in the line specs of diff
568 lines.append({
579 lines.append({
569 'old_lineno': '...',
580 'old_lineno': '...',
570 'new_lineno': '...',
581 'new_lineno': '...',
571 'action': Action.CONTEXT,
582 'action': Action.CONTEXT,
572 'line': self._clean_line(line, command)
583 'line': self._clean_line(line, command)
573 })
584 })
574
585
575 except StopIteration:
586 except StopIteration:
576 pass
587 pass
577 return ''.join(raw_diff), chunks, stats
588 return ''.join(raw_diff), chunks, stats
578
589
579 def _safe_id(self, idstring):
590 def _safe_id(self, idstring):
580 """Make a string safe for including in an id attribute.
591 """Make a string safe for including in an id attribute.
581
592
582 The HTML spec says that id attributes 'must begin with
593 The HTML spec says that id attributes 'must begin with
583 a letter ([A-Za-z]) and may be followed by any number
594 a letter ([A-Za-z]) and may be followed by any number
584 of letters, digits ([0-9]), hyphens ("-"), underscores
595 of letters, digits ([0-9]), hyphens ("-"), underscores
585 ("_"), colons (":"), and periods (".")'. These regexps
596 ("_"), colons (":"), and periods (".")'. These regexps
586 are slightly over-zealous, in that they remove colons
597 are slightly over-zealous, in that they remove colons
587 and periods unnecessarily.
598 and periods unnecessarily.
588
599
589 Whitespace is transformed into underscores, and then
600 Whitespace is transformed into underscores, and then
590 anything which is not a hyphen or a character that
601 anything which is not a hyphen or a character that
591 matches \w (alphanumerics and underscore) is removed.
602 matches \w (alphanumerics and underscore) is removed.
592
603
593 """
604 """
594 # Transform all whitespace to underscore
605 # Transform all whitespace to underscore
595 idstring = re.sub(r'\s', "_", '%s' % idstring)
606 idstring = re.sub(r'\s', "_", '%s' % idstring)
596 # Remove everything that is not a hyphen or a member of \w
607 # Remove everything that is not a hyphen or a member of \w
597 idstring = re.sub(r'(?!-)\W', "", idstring).lower()
608 idstring = re.sub(r'(?!-)\W', "", idstring).lower()
598 return idstring
609 return idstring
599
610
600 def prepare(self, inline_diff=True):
611 def prepare(self, inline_diff=True):
601 """
612 """
602 Prepare the passed udiff for HTML rendering.
613 Prepare the passed udiff for HTML rendering.
603
614
604 :return: A list of dicts with diff information.
615 :return: A list of dicts with diff information.
605 """
616 """
606 parsed = self._parser(inline_diff=inline_diff)
617 parsed = self._parser(inline_diff=inline_diff)
607 self.parsed = True
618 self.parsed = True
608 self.parsed_diff = parsed
619 self.parsed_diff = parsed
609 return parsed
620 return parsed
610
621
611 def as_raw(self, diff_lines=None):
622 def as_raw(self, diff_lines=None):
612 """
623 """
613 Returns raw diff as a byte string
624 Returns raw diff as a byte string
614 """
625 """
615 return self._diff.raw
626 return self._diff.raw
616
627
617 def as_html(self, table_class='code-difftable', line_class='line',
628 def as_html(self, table_class='code-difftable', line_class='line',
618 old_lineno_class='lineno old', new_lineno_class='lineno new',
629 old_lineno_class='lineno old', new_lineno_class='lineno new',
619 code_class='code', enable_comments=False, parsed_lines=None):
630 code_class='code', enable_comments=False, parsed_lines=None):
620 """
631 """
621 Return given diff as html table with customized css classes
632 Return given diff as html table with customized css classes
622 """
633 """
623 def _link_to_if(condition, label, url):
634 def _link_to_if(condition, label, url):
624 """
635 """
625 Generates a link if condition is meet or just the label if not.
636 Generates a link if condition is meet or just the label if not.
626 """
637 """
627
638
628 if condition:
639 if condition:
629 return '''<a href="%(url)s" class="tooltip"
640 return '''<a href="%(url)s" class="tooltip"
630 title="%(title)s">%(label)s</a>''' % {
641 title="%(title)s">%(label)s</a>''' % {
631 'title': _('Click to select line'),
642 'title': _('Click to select line'),
632 'url': url,
643 'url': url,
633 'label': label
644 'label': label
634 }
645 }
635 else:
646 else:
636 return label
647 return label
637 if not self.parsed:
648 if not self.parsed:
638 self.prepare()
649 self.prepare()
639
650
640 diff_lines = self.parsed_diff
651 diff_lines = self.parsed_diff
641 if parsed_lines:
652 if parsed_lines:
642 diff_lines = parsed_lines
653 diff_lines = parsed_lines
643
654
644 _html_empty = True
655 _html_empty = True
645 _html = []
656 _html = []
646 _html.append('''<table class="%(table_class)s">\n''' % {
657 _html.append('''<table class="%(table_class)s">\n''' % {
647 'table_class': table_class
658 'table_class': table_class
648 })
659 })
649
660
650 for diff in diff_lines:
661 for diff in diff_lines:
651 for line in diff['chunks']:
662 for line in diff['chunks']:
652 _html_empty = False
663 _html_empty = False
653 for change in line:
664 for change in line:
654 _html.append('''<tr class="%(lc)s %(action)s">\n''' % {
665 _html.append('''<tr class="%(lc)s %(action)s">\n''' % {
655 'lc': line_class,
666 'lc': line_class,
656 'action': change['action']
667 'action': change['action']
657 })
668 })
658 anchor_old_id = ''
669 anchor_old_id = ''
659 anchor_new_id = ''
670 anchor_new_id = ''
660 anchor_old = "%(filename)s_o%(oldline_no)s" % {
671 anchor_old = "%(filename)s_o%(oldline_no)s" % {
661 'filename': self._safe_id(diff['filename']),
672 'filename': self._safe_id(diff['filename']),
662 'oldline_no': change['old_lineno']
673 'oldline_no': change['old_lineno']
663 }
674 }
664 anchor_new = "%(filename)s_n%(oldline_no)s" % {
675 anchor_new = "%(filename)s_n%(oldline_no)s" % {
665 'filename': self._safe_id(diff['filename']),
676 'filename': self._safe_id(diff['filename']),
666 'oldline_no': change['new_lineno']
677 'oldline_no': change['new_lineno']
667 }
678 }
668 cond_old = (change['old_lineno'] != '...' and
679 cond_old = (change['old_lineno'] != '...' and
669 change['old_lineno'])
680 change['old_lineno'])
670 cond_new = (change['new_lineno'] != '...' and
681 cond_new = (change['new_lineno'] != '...' and
671 change['new_lineno'])
682 change['new_lineno'])
672 if cond_old:
683 if cond_old:
673 anchor_old_id = 'id="%s"' % anchor_old
684 anchor_old_id = 'id="%s"' % anchor_old
674 if cond_new:
685 if cond_new:
675 anchor_new_id = 'id="%s"' % anchor_new
686 anchor_new_id = 'id="%s"' % anchor_new
676
687
677 if change['action'] != Action.CONTEXT:
688 if change['action'] != Action.CONTEXT:
678 anchor_link = True
689 anchor_link = True
679 else:
690 else:
680 anchor_link = False
691 anchor_link = False
681
692
682 ###########################################################
693 ###########################################################
683 # COMMENT ICON
694 # COMMENT ICON
684 ###########################################################
695 ###########################################################
685 _html.append('''\t<td class="add-comment-line"><span class="add-comment-content">''')
696 _html.append('''\t<td class="add-comment-line"><span class="add-comment-content">''')
686
697
687 if enable_comments and change['action'] != Action.CONTEXT:
698 if enable_comments and change['action'] != Action.CONTEXT:
688 _html.append('''<a href="#"><span class="icon-comment-add"></span></a>''')
699 _html.append('''<a href="#"><span class="icon-comment-add"></span></a>''')
689
700
690 _html.append('''</span></td>\n''')
701 _html.append('''</span></td>\n''')
691
702
692 ###########################################################
703 ###########################################################
693 # OLD LINE NUMBER
704 # OLD LINE NUMBER
694 ###########################################################
705 ###########################################################
695 _html.append('''\t<td %(a_id)s class="%(olc)s">''' % {
706 _html.append('''\t<td %(a_id)s class="%(olc)s">''' % {
696 'a_id': anchor_old_id,
707 'a_id': anchor_old_id,
697 'olc': old_lineno_class
708 'olc': old_lineno_class
698 })
709 })
699
710
700 _html.append('''%(link)s''' % {
711 _html.append('''%(link)s''' % {
701 'link': _link_to_if(anchor_link, change['old_lineno'],
712 'link': _link_to_if(anchor_link, change['old_lineno'],
702 '#%s' % anchor_old)
713 '#%s' % anchor_old)
703 })
714 })
704 _html.append('''</td>\n''')
715 _html.append('''</td>\n''')
705 ###########################################################
716 ###########################################################
706 # NEW LINE NUMBER
717 # NEW LINE NUMBER
707 ###########################################################
718 ###########################################################
708
719
709 _html.append('''\t<td %(a_id)s class="%(nlc)s">''' % {
720 _html.append('''\t<td %(a_id)s class="%(nlc)s">''' % {
710 'a_id': anchor_new_id,
721 'a_id': anchor_new_id,
711 'nlc': new_lineno_class
722 'nlc': new_lineno_class
712 })
723 })
713
724
714 _html.append('''%(link)s''' % {
725 _html.append('''%(link)s''' % {
715 'link': _link_to_if(anchor_link, change['new_lineno'],
726 'link': _link_to_if(anchor_link, change['new_lineno'],
716 '#%s' % anchor_new)
727 '#%s' % anchor_new)
717 })
728 })
718 _html.append('''</td>\n''')
729 _html.append('''</td>\n''')
719 ###########################################################
730 ###########################################################
720 # CODE
731 # CODE
721 ###########################################################
732 ###########################################################
722 code_classes = [code_class]
733 code_classes = [code_class]
723 if (not enable_comments or
734 if (not enable_comments or
724 change['action'] == Action.CONTEXT):
735 change['action'] == Action.CONTEXT):
725 code_classes.append('no-comment')
736 code_classes.append('no-comment')
726 _html.append('\t<td class="%s">' % ' '.join(code_classes))
737 _html.append('\t<td class="%s">' % ' '.join(code_classes))
727 _html.append('''\n\t\t<pre>%(code)s</pre>\n''' % {
738 _html.append('''\n\t\t<pre>%(code)s</pre>\n''' % {
728 'code': change['line']
739 'code': change['line']
729 })
740 })
730
741
731 _html.append('''\t</td>''')
742 _html.append('''\t</td>''')
732 _html.append('''\n</tr>\n''')
743 _html.append('''\n</tr>\n''')
733 _html.append('''</table>''')
744 _html.append('''</table>''')
734 if _html_empty:
745 if _html_empty:
735 return None
746 return None
736 return ''.join(_html)
747 return ''.join(_html)
737
748
738 def stat(self):
749 def stat(self):
739 """
750 """
740 Returns tuple of added, and removed lines for this instance
751 Returns tuple of added, and removed lines for this instance
741 """
752 """
742 return self.adds, self.removes
753 return self.adds, self.removes
743
754
744 def get_context_of_line(
755 def get_context_of_line(
745 self, path, diff_line=None, context_before=3, context_after=3):
756 self, path, diff_line=None, context_before=3, context_after=3):
746 """
757 """
747 Returns the context lines for the specified diff line.
758 Returns the context lines for the specified diff line.
748
759
749 :type diff_line: :class:`DiffLineNumber`
760 :type diff_line: :class:`DiffLineNumber`
750 """
761 """
751 assert self.parsed, "DiffProcessor is not initialized."
762 assert self.parsed, "DiffProcessor is not initialized."
752
763
753 if None not in diff_line:
764 if None not in diff_line:
754 raise ValueError(
765 raise ValueError(
755 "Cannot specify both line numbers: {}".format(diff_line))
766 "Cannot specify both line numbers: {}".format(diff_line))
756
767
757 file_diff = self._get_file_diff(path)
768 file_diff = self._get_file_diff(path)
758 chunk, idx = self._find_chunk_line_index(file_diff, diff_line)
769 chunk, idx = self._find_chunk_line_index(file_diff, diff_line)
759
770
760 first_line_to_include = max(idx - context_before, 0)
771 first_line_to_include = max(idx - context_before, 0)
761 first_line_after_context = idx + context_after + 1
772 first_line_after_context = idx + context_after + 1
762 context_lines = chunk[first_line_to_include:first_line_after_context]
773 context_lines = chunk[first_line_to_include:first_line_after_context]
763
774
764 line_contents = [
775 line_contents = [
765 _context_line(line) for line in context_lines
776 _context_line(line) for line in context_lines
766 if _is_diff_content(line)]
777 if _is_diff_content(line)]
767 # TODO: johbo: Interim fixup, the diff chunks drop the final newline.
778 # TODO: johbo: Interim fixup, the diff chunks drop the final newline.
768 # Once they are fixed, we can drop this line here.
779 # Once they are fixed, we can drop this line here.
769 if line_contents:
780 if line_contents:
770 line_contents[-1] = (
781 line_contents[-1] = (
771 line_contents[-1][0], line_contents[-1][1].rstrip('\n') + '\n')
782 line_contents[-1][0], line_contents[-1][1].rstrip('\n') + '\n')
772 return line_contents
783 return line_contents
773
784
774 def find_context(self, path, context, offset=0):
785 def find_context(self, path, context, offset=0):
775 """
786 """
776 Finds the given `context` inside of the diff.
787 Finds the given `context` inside of the diff.
777
788
778 Use the parameter `offset` to specify which offset the target line has
789 Use the parameter `offset` to specify which offset the target line has
779 inside of the given `context`. This way the correct diff line will be
790 inside of the given `context`. This way the correct diff line will be
780 returned.
791 returned.
781
792
782 :param offset: Shall be used to specify the offset of the main line
793 :param offset: Shall be used to specify the offset of the main line
783 within the given `context`.
794 within the given `context`.
784 """
795 """
785 if offset < 0 or offset >= len(context):
796 if offset < 0 or offset >= len(context):
786 raise ValueError(
797 raise ValueError(
787 "Only positive values up to the length of the context "
798 "Only positive values up to the length of the context "
788 "minus one are allowed.")
799 "minus one are allowed.")
789
800
790 matches = []
801 matches = []
791 file_diff = self._get_file_diff(path)
802 file_diff = self._get_file_diff(path)
792
803
793 for chunk in file_diff['chunks']:
804 for chunk in file_diff['chunks']:
794 context_iter = iter(context)
805 context_iter = iter(context)
795 for line_idx, line in enumerate(chunk):
806 for line_idx, line in enumerate(chunk):
796 try:
807 try:
797 if _context_line(line) == context_iter.next():
808 if _context_line(line) == context_iter.next():
798 continue
809 continue
799 except StopIteration:
810 except StopIteration:
800 matches.append((line_idx, chunk))
811 matches.append((line_idx, chunk))
801 context_iter = iter(context)
812 context_iter = iter(context)
802
813
803 # Increment position and triger StopIteration
814 # Increment position and triger StopIteration
804 # if we had a match at the end
815 # if we had a match at the end
805 line_idx += 1
816 line_idx += 1
806 try:
817 try:
807 context_iter.next()
818 context_iter.next()
808 except StopIteration:
819 except StopIteration:
809 matches.append((line_idx, chunk))
820 matches.append((line_idx, chunk))
810
821
811 effective_offset = len(context) - offset
822 effective_offset = len(context) - offset
812 found_at_diff_lines = [
823 found_at_diff_lines = [
813 _line_to_diff_line_number(chunk[idx - effective_offset])
824 _line_to_diff_line_number(chunk[idx - effective_offset])
814 for idx, chunk in matches]
825 for idx, chunk in matches]
815
826
816 return found_at_diff_lines
827 return found_at_diff_lines
817
828
818 def _get_file_diff(self, path):
829 def _get_file_diff(self, path):
819 for file_diff in self.parsed_diff:
830 for file_diff in self.parsed_diff:
820 if file_diff['filename'] == path:
831 if file_diff['filename'] == path:
821 break
832 break
822 else:
833 else:
823 raise FileNotInDiffException("File {} not in diff".format(path))
834 raise FileNotInDiffException("File {} not in diff".format(path))
824 return file_diff
835 return file_diff
825
836
826 def _find_chunk_line_index(self, file_diff, diff_line):
837 def _find_chunk_line_index(self, file_diff, diff_line):
827 for chunk in file_diff['chunks']:
838 for chunk in file_diff['chunks']:
828 for idx, line in enumerate(chunk):
839 for idx, line in enumerate(chunk):
829 if line['old_lineno'] == diff_line.old:
840 if line['old_lineno'] == diff_line.old:
830 return chunk, idx
841 return chunk, idx
831 if line['new_lineno'] == diff_line.new:
842 if line['new_lineno'] == diff_line.new:
832 return chunk, idx
843 return chunk, idx
833 raise LineNotInDiffException(
844 raise LineNotInDiffException(
834 "The line {} is not part of the diff.".format(diff_line))
845 "The line {} is not part of the diff.".format(diff_line))
835
846
836
847
837 def _is_diff_content(line):
848 def _is_diff_content(line):
838 return line['action'] in (
849 return line['action'] in (
839 Action.UNMODIFIED, Action.ADD, Action.DELETE)
850 Action.UNMODIFIED, Action.ADD, Action.DELETE)
840
851
841
852
842 def _context_line(line):
853 def _context_line(line):
843 return (line['action'], line['line'])
854 return (line['action'], line['line'])
844
855
845
856
846 DiffLineNumber = collections.namedtuple('DiffLineNumber', ['old', 'new'])
857 DiffLineNumber = collections.namedtuple('DiffLineNumber', ['old', 'new'])
847
858
848
859
849 def _line_to_diff_line_number(line):
860 def _line_to_diff_line_number(line):
850 new_line_no = line['new_lineno'] or None
861 new_line_no = line['new_lineno'] or None
851 old_line_no = line['old_lineno'] or None
862 old_line_no = line['old_lineno'] or None
852 return DiffLineNumber(old=old_line_no, new=new_line_no)
863 return DiffLineNumber(old=old_line_no, new=new_line_no)
853
864
854
865
855 class FileNotInDiffException(Exception):
866 class FileNotInDiffException(Exception):
856 """
867 """
857 Raised when the context for a missing file is requested.
868 Raised when the context for a missing file is requested.
858
869
859 If you request the context for a line in a file which is not part of the
870 If you request the context for a line in a file which is not part of the
860 given diff, then this exception is raised.
871 given diff, then this exception is raised.
861 """
872 """
862
873
863
874
864 class LineNotInDiffException(Exception):
875 class LineNotInDiffException(Exception):
865 """
876 """
866 Raised when the context for a missing line is requested.
877 Raised when the context for a missing line is requested.
867
878
868 If you request the context for a line in a file and this line is not
879 If you request the context for a line in a file and this line is not
869 part of the given diff, then this exception is raised.
880 part of the given diff, then this exception is raised.
870 """
881 """
871
882
872
883
873 class DiffLimitExceeded(Exception):
884 class DiffLimitExceeded(Exception):
874 pass
885 pass
@@ -1,274 +1,279 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2016 RhodeCode GmbH
3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import json
21 import json
22 import logging
22 import logging
23 import urlparse
23 import urlparse
24 import threading
24 import threading
25 from BaseHTTPServer import BaseHTTPRequestHandler
25 from BaseHTTPServer import BaseHTTPRequestHandler
26 from SocketServer import TCPServer
26 from SocketServer import TCPServer
27 from routes.util import URLGenerator
27 from routes.util import URLGenerator
28
28
29 import Pyro4
29 import Pyro4
30 import pylons
30 import pylons
31 import rhodecode
31 import rhodecode
32
32
33 from rhodecode.model import meta
33 from rhodecode.lib import hooks_base
34 from rhodecode.lib import hooks_base
34 from rhodecode.lib.utils2 import (
35 from rhodecode.lib.utils2 import (
35 AttributeDict, safe_str, get_routes_generator_for_server_url)
36 AttributeDict, safe_str, get_routes_generator_for_server_url)
36
37
37
38
38 log = logging.getLogger(__name__)
39 log = logging.getLogger(__name__)
39
40
40
41
41 class HooksHttpHandler(BaseHTTPRequestHandler):
42 class HooksHttpHandler(BaseHTTPRequestHandler):
42 def do_POST(self):
43 def do_POST(self):
43 method, extras = self._read_request()
44 method, extras = self._read_request()
44 try:
45 try:
45 result = self._call_hook(method, extras)
46 result = self._call_hook(method, extras)
46 except Exception as e:
47 except Exception as e:
47 result = {
48 result = {
48 'exception': e.__class__.__name__,
49 'exception': e.__class__.__name__,
49 'exception_args': e.args
50 'exception_args': e.args
50 }
51 }
51 self._write_response(result)
52 self._write_response(result)
52
53
53 def _read_request(self):
54 def _read_request(self):
54 length = int(self.headers['Content-Length'])
55 length = int(self.headers['Content-Length'])
55 body = self.rfile.read(length).decode('utf-8')
56 body = self.rfile.read(length).decode('utf-8')
56 data = json.loads(body)
57 data = json.loads(body)
57 return data['method'], data['extras']
58 return data['method'], data['extras']
58
59
59 def _write_response(self, result):
60 def _write_response(self, result):
60 self.send_response(200)
61 self.send_response(200)
61 self.send_header("Content-type", "text/json")
62 self.send_header("Content-type", "text/json")
62 self.end_headers()
63 self.end_headers()
63 self.wfile.write(json.dumps(result))
64 self.wfile.write(json.dumps(result))
64
65
65 def _call_hook(self, method, extras):
66 def _call_hook(self, method, extras):
66 hooks = Hooks()
67 hooks = Hooks()
68 try:
67 result = getattr(hooks, method)(extras)
69 result = getattr(hooks, method)(extras)
70 finally:
71 meta.Session.remove()
68 return result
72 return result
69
73
70 def log_message(self, format, *args):
74 def log_message(self, format, *args):
71 """
75 """
72 This is an overriden method of BaseHTTPRequestHandler which logs using
76 This is an overriden method of BaseHTTPRequestHandler which logs using
73 logging library instead of writing directly to stderr.
77 logging library instead of writing directly to stderr.
74 """
78 """
75
79
76 message = format % args
80 message = format % args
77
81
78 # TODO: mikhail: add different log levels support
82 # TODO: mikhail: add different log levels support
79 log.debug(
83 log.debug(
80 "%s - - [%s] %s", self.client_address[0],
84 "%s - - [%s] %s", self.client_address[0],
81 self.log_date_time_string(), message)
85 self.log_date_time_string(), message)
82
86
83
87
84 class DummyHooksCallbackDaemon(object):
88 class DummyHooksCallbackDaemon(object):
85 def __init__(self):
89 def __init__(self):
86 self.hooks_module = Hooks.__module__
90 self.hooks_module = Hooks.__module__
87
91
88 def __enter__(self):
92 def __enter__(self):
89 log.debug('Running dummy hooks callback daemon')
93 log.debug('Running dummy hooks callback daemon')
90 return self
94 return self
91
95
92 def __exit__(self, exc_type, exc_val, exc_tb):
96 def __exit__(self, exc_type, exc_val, exc_tb):
93 log.debug('Exiting dummy hooks callback daemon')
97 log.debug('Exiting dummy hooks callback daemon')
94
98
95
99
96 class ThreadedHookCallbackDaemon(object):
100 class ThreadedHookCallbackDaemon(object):
97
101
98 _callback_thread = None
102 _callback_thread = None
99 _daemon = None
103 _daemon = None
100 _done = False
104 _done = False
101
105
102 def __init__(self):
106 def __init__(self):
103 self._prepare()
107 self._prepare()
104
108
105 def __enter__(self):
109 def __enter__(self):
106 self._run()
110 self._run()
107 return self
111 return self
108
112
109 def __exit__(self, exc_type, exc_val, exc_tb):
113 def __exit__(self, exc_type, exc_val, exc_tb):
110 self._stop()
114 self._stop()
111
115
112 def _prepare(self):
116 def _prepare(self):
113 raise NotImplementedError()
117 raise NotImplementedError()
114
118
115 def _run(self):
119 def _run(self):
116 raise NotImplementedError()
120 raise NotImplementedError()
117
121
118 def _stop(self):
122 def _stop(self):
119 raise NotImplementedError()
123 raise NotImplementedError()
120
124
121
125
122 class Pyro4HooksCallbackDaemon(ThreadedHookCallbackDaemon):
126 class Pyro4HooksCallbackDaemon(ThreadedHookCallbackDaemon):
123 """
127 """
124 Context manager which will run a callback daemon in a background thread.
128 Context manager which will run a callback daemon in a background thread.
125 """
129 """
126
130
127 hooks_uri = None
131 hooks_uri = None
128
132
129 def _prepare(self):
133 def _prepare(self):
130 log.debug("Preparing callback daemon and registering hook object")
134 log.debug("Preparing callback daemon and registering hook object")
131 self._daemon = Pyro4.Daemon()
135 self._daemon = Pyro4.Daemon()
132 hooks_interface = Hooks()
136 hooks_interface = Hooks()
133 self.hooks_uri = str(self._daemon.register(hooks_interface))
137 self.hooks_uri = str(self._daemon.register(hooks_interface))
134 log.debug("Hooks uri is: %s", self.hooks_uri)
138 log.debug("Hooks uri is: %s", self.hooks_uri)
135
139
136 def _run(self):
140 def _run(self):
137 log.debug("Running event loop of callback daemon in background thread")
141 log.debug("Running event loop of callback daemon in background thread")
138 callback_thread = threading.Thread(
142 callback_thread = threading.Thread(
139 target=self._daemon.requestLoop,
143 target=self._daemon.requestLoop,
140 kwargs={'loopCondition': lambda: not self._done})
144 kwargs={'loopCondition': lambda: not self._done})
141 callback_thread.daemon = True
145 callback_thread.daemon = True
142 callback_thread.start()
146 callback_thread.start()
143 self._callback_thread = callback_thread
147 self._callback_thread = callback_thread
144
148
145 def _stop(self):
149 def _stop(self):
146 log.debug("Waiting for background thread to finish.")
150 log.debug("Waiting for background thread to finish.")
147 self._done = True
151 self._done = True
148 self._callback_thread.join()
152 self._callback_thread.join()
149 self._daemon.close()
153 self._daemon.close()
150 self._daemon = None
154 self._daemon = None
151 self._callback_thread = None
155 self._callback_thread = None
152
156
153
157
154 class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
158 class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
155 """
159 """
156 Context manager which will run a callback daemon in a background thread.
160 Context manager which will run a callback daemon in a background thread.
157 """
161 """
158
162
159 hooks_uri = None
163 hooks_uri = None
160
164
161 IP_ADDRESS = '127.0.0.1'
165 IP_ADDRESS = '127.0.0.1'
162
166
163 # From Python docs: Polling reduces our responsiveness to a shutdown
167 # From Python docs: Polling reduces our responsiveness to a shutdown
164 # request and wastes cpu at all other times.
168 # request and wastes cpu at all other times.
165 POLL_INTERVAL = 0.1
169 POLL_INTERVAL = 0.1
166
170
167 def _prepare(self):
171 def _prepare(self):
168 log.debug("Preparing callback daemon and registering hook object")
172 log.debug("Preparing callback daemon and registering hook object")
169
173
170 self._done = False
174 self._done = False
171 self._daemon = TCPServer((self.IP_ADDRESS, 0), HooksHttpHandler)
175 self._daemon = TCPServer((self.IP_ADDRESS, 0), HooksHttpHandler)
172 _, port = self._daemon.server_address
176 _, port = self._daemon.server_address
173 self.hooks_uri = '{}:{}'.format(self.IP_ADDRESS, port)
177 self.hooks_uri = '{}:{}'.format(self.IP_ADDRESS, port)
174
178
175 log.debug("Hooks uri is: %s", self.hooks_uri)
179 log.debug("Hooks uri is: %s", self.hooks_uri)
176
180
177 def _run(self):
181 def _run(self):
178 log.debug("Running event loop of callback daemon in background thread")
182 log.debug("Running event loop of callback daemon in background thread")
179 callback_thread = threading.Thread(
183 callback_thread = threading.Thread(
180 target=self._daemon.serve_forever,
184 target=self._daemon.serve_forever,
181 kwargs={'poll_interval': self.POLL_INTERVAL})
185 kwargs={'poll_interval': self.POLL_INTERVAL})
182 callback_thread.daemon = True
186 callback_thread.daemon = True
183 callback_thread.start()
187 callback_thread.start()
184 self._callback_thread = callback_thread
188 self._callback_thread = callback_thread
185
189
186 def _stop(self):
190 def _stop(self):
187 log.debug("Waiting for background thread to finish.")
191 log.debug("Waiting for background thread to finish.")
188 self._daemon.shutdown()
192 self._daemon.shutdown()
189 self._callback_thread.join()
193 self._callback_thread.join()
190 self._daemon = None
194 self._daemon = None
191 self._callback_thread = None
195 self._callback_thread = None
192
196
193
197
194 def prepare_callback_daemon(extras, protocol=None, use_direct_calls=False):
198 def prepare_callback_daemon(extras, protocol=None, use_direct_calls=False):
195 callback_daemon = None
199 callback_daemon = None
196 protocol = protocol.lower() if protocol else None
200 protocol = protocol.lower() if protocol else None
197
201
198 if use_direct_calls:
202 if use_direct_calls:
199 callback_daemon = DummyHooksCallbackDaemon()
203 callback_daemon = DummyHooksCallbackDaemon()
200 extras['hooks_module'] = callback_daemon.hooks_module
204 extras['hooks_module'] = callback_daemon.hooks_module
201 else:
205 else:
202 if protocol == 'pyro4':
206 if protocol == 'pyro4':
203 callback_daemon = Pyro4HooksCallbackDaemon()
207 callback_daemon = Pyro4HooksCallbackDaemon()
204 elif protocol == 'http':
208 elif protocol == 'http':
205 callback_daemon = HttpHooksCallbackDaemon()
209 callback_daemon = HttpHooksCallbackDaemon()
206 else:
210 else:
207 log.error('Unsupported callback daemon protocol "%s"', protocol)
211 log.error('Unsupported callback daemon protocol "%s"', protocol)
208 raise Exception('Unsupported callback daemon protocol.')
212 raise Exception('Unsupported callback daemon protocol.')
209
213
210 extras['hooks_uri'] = callback_daemon.hooks_uri
214 extras['hooks_uri'] = callback_daemon.hooks_uri
211 extras['hooks_protocol'] = protocol
215 extras['hooks_protocol'] = protocol
212
216
213 return callback_daemon, extras
217 return callback_daemon, extras
214
218
215
219
216 class Hooks(object):
220 class Hooks(object):
217 """
221 """
218 Exposes the hooks for remote call backs
222 Exposes the hooks for remote call backs
219 """
223 """
220
224
221 @Pyro4.callback
225 @Pyro4.callback
222 def repo_size(self, extras):
226 def repo_size(self, extras):
223 log.debug("Called repo_size of Hooks object")
227 log.debug("Called repo_size of Hooks object")
224 return self._call_hook(hooks_base.repo_size, extras)
228 return self._call_hook(hooks_base.repo_size, extras)
225
229
226 @Pyro4.callback
230 @Pyro4.callback
227 def pre_pull(self, extras):
231 def pre_pull(self, extras):
228 log.debug("Called pre_pull of Hooks object")
232 log.debug("Called pre_pull of Hooks object")
229 return self._call_hook(hooks_base.pre_pull, extras)
233 return self._call_hook(hooks_base.pre_pull, extras)
230
234
231 @Pyro4.callback
235 @Pyro4.callback
232 def post_pull(self, extras):
236 def post_pull(self, extras):
233 log.debug("Called post_pull of Hooks object")
237 log.debug("Called post_pull of Hooks object")
234 return self._call_hook(hooks_base.post_pull, extras)
238 return self._call_hook(hooks_base.post_pull, extras)
235
239
236 @Pyro4.callback
240 @Pyro4.callback
237 def pre_push(self, extras):
241 def pre_push(self, extras):
238 log.debug("Called pre_push of Hooks object")
242 log.debug("Called pre_push of Hooks object")
239 return self._call_hook(hooks_base.pre_push, extras)
243 return self._call_hook(hooks_base.pre_push, extras)
240
244
241 @Pyro4.callback
245 @Pyro4.callback
242 def post_push(self, extras):
246 def post_push(self, extras):
243 log.debug("Called post_push of Hooks object")
247 log.debug("Called post_push of Hooks object")
244 return self._call_hook(hooks_base.post_push, extras)
248 return self._call_hook(hooks_base.post_push, extras)
245
249
246 def _call_hook(self, hook, extras):
250 def _call_hook(self, hook, extras):
247 extras = AttributeDict(extras)
251 extras = AttributeDict(extras)
248 pylons_router = get_routes_generator_for_server_url(extras.server_url)
252 pylons_router = get_routes_generator_for_server_url(extras.server_url)
249 pylons.url._push_object(pylons_router)
253 pylons.url._push_object(pylons_router)
250
254
251 try:
255 try:
252 result = hook(extras)
256 result = hook(extras)
253 except Exception as error:
257 except Exception as error:
254 log.exception('Exception when handling hook %s', hook)
258 log.exception('Exception when handling hook %s', hook)
255 error_args = error.args
259 error_args = error.args
256 return {
260 return {
257 'status': 128,
261 'status': 128,
258 'output': '',
262 'output': '',
259 'exception': type(error).__name__,
263 'exception': type(error).__name__,
260 'exception_args': error_args,
264 'exception_args': error_args,
261 }
265 }
262 finally:
266 finally:
263 pylons.url._pop_object()
267 pylons.url._pop_object()
268 meta.Session.remove()
264
269
265 return {
270 return {
266 'status': result.status,
271 'status': result.status,
267 'output': result.output,
272 'output': result.output,
268 }
273 }
269
274
270 def __enter__(self):
275 def __enter__(self):
271 return self
276 return self
272
277
273 def __exit__(self, exc_type, exc_val, exc_tb):
278 def __exit__(self, exc_type, exc_val, exc_tb):
274 pass
279 pass
@@ -1,445 +1,448 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2014-2016 RhodeCode GmbH
3 # Copyright (C) 2014-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 SimpleVCS middleware for handling protocol request (push/clone etc.)
22 SimpleVCS middleware for handling protocol request (push/clone etc.)
23 It's implemented with basic auth function
23 It's implemented with basic auth function
24 """
24 """
25
25
26 import os
26 import os
27 import logging
27 import logging
28 import importlib
28 import importlib
29 from functools import wraps
29 from functools import wraps
30
30
31 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
31 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
32 from webob.exc import (
32 from webob.exc import (
33 HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
33 HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
34
34
35 import rhodecode
35 import rhodecode
36 from rhodecode.authentication.base import authenticate, VCS_TYPE
36 from rhodecode.authentication.base import authenticate, VCS_TYPE
37 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
37 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
38 from rhodecode.lib.base import BasicAuth, get_ip_addr, vcs_operation_context
38 from rhodecode.lib.base import BasicAuth, get_ip_addr, vcs_operation_context
39 from rhodecode.lib.exceptions import (
39 from rhodecode.lib.exceptions import (
40 HTTPLockedRC, HTTPRequirementError, UserCreationError,
40 HTTPLockedRC, HTTPRequirementError, UserCreationError,
41 NotAllowedToCreateUserError)
41 NotAllowedToCreateUserError)
42 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
42 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
43 from rhodecode.lib.middleware import appenlight
43 from rhodecode.lib.middleware import appenlight
44 from rhodecode.lib.middleware.utils import scm_app
44 from rhodecode.lib.middleware.utils import scm_app
45 from rhodecode.lib.utils import (
45 from rhodecode.lib.utils import (
46 is_valid_repo, get_rhodecode_realm, get_rhodecode_base_path)
46 is_valid_repo, get_rhodecode_realm, get_rhodecode_base_path)
47 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool
47 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool
48 from rhodecode.lib.vcs.conf import settings as vcs_settings
48 from rhodecode.lib.vcs.conf import settings as vcs_settings
49 from rhodecode.model import meta
49 from rhodecode.model import meta
50 from rhodecode.model.db import User, Repository
50 from rhodecode.model.db import User, Repository
51 from rhodecode.model.scm import ScmModel
51 from rhodecode.model.scm import ScmModel
52 from rhodecode.model.settings import SettingsModel
52 from rhodecode.model.settings import SettingsModel
53
53
54 log = logging.getLogger(__name__)
54 log = logging.getLogger(__name__)
55
55
56
56
57 def initialize_generator(factory):
57 def initialize_generator(factory):
58 """
58 """
59 Initializes the returned generator by draining its first element.
59 Initializes the returned generator by draining its first element.
60
60
61 This can be used to give a generator an initializer, which is the code
61 This can be used to give a generator an initializer, which is the code
62 up to the first yield statement. This decorator enforces that the first
62 up to the first yield statement. This decorator enforces that the first
63 produced element has the value ``"__init__"`` to make its special
63 produced element has the value ``"__init__"`` to make its special
64 purpose very explicit in the using code.
64 purpose very explicit in the using code.
65 """
65 """
66
66
67 @wraps(factory)
67 @wraps(factory)
68 def wrapper(*args, **kwargs):
68 def wrapper(*args, **kwargs):
69 gen = factory(*args, **kwargs)
69 gen = factory(*args, **kwargs)
70 try:
70 try:
71 init = gen.next()
71 init = gen.next()
72 except StopIteration:
72 except StopIteration:
73 raise ValueError('Generator must yield at least one element.')
73 raise ValueError('Generator must yield at least one element.')
74 if init != "__init__":
74 if init != "__init__":
75 raise ValueError('First yielded element must be "__init__".')
75 raise ValueError('First yielded element must be "__init__".')
76 return gen
76 return gen
77 return wrapper
77 return wrapper
78
78
79
79
80 class SimpleVCS(object):
80 class SimpleVCS(object):
81 """Common functionality for SCM HTTP handlers."""
81 """Common functionality for SCM HTTP handlers."""
82
82
83 SCM = 'unknown'
83 SCM = 'unknown'
84
84
85 def __init__(self, application, config, registry):
85 def __init__(self, application, config, registry):
86 self.registry = registry
86 self.registry = registry
87 self.application = application
87 self.application = application
88 self.config = config
88 self.config = config
89 # base path of repo locations
89 # base path of repo locations
90 self.basepath = get_rhodecode_base_path()
90 self.basepath = get_rhodecode_base_path()
91 # authenticate this VCS request using authfunc
91 # authenticate this VCS request using authfunc
92 auth_ret_code_detection = \
92 auth_ret_code_detection = \
93 str2bool(self.config.get('auth_ret_code_detection', False))
93 str2bool(self.config.get('auth_ret_code_detection', False))
94 self.authenticate = BasicAuth(
94 self.authenticate = BasicAuth(
95 '', authenticate, registry, config.get('auth_ret_code'),
95 '', authenticate, registry, config.get('auth_ret_code'),
96 auth_ret_code_detection)
96 auth_ret_code_detection)
97 self.ip_addr = '0.0.0.0'
97 self.ip_addr = '0.0.0.0'
98
98
99 @property
99 @property
100 def scm_app(self):
100 def scm_app(self):
101 custom_implementation = self.config.get('vcs.scm_app_implementation')
101 custom_implementation = self.config.get('vcs.scm_app_implementation')
102 if custom_implementation and custom_implementation != 'pyro4':
102 if custom_implementation and custom_implementation != 'pyro4':
103 log.info(
103 log.info(
104 "Using custom implementation of scm_app: %s",
104 "Using custom implementation of scm_app: %s",
105 custom_implementation)
105 custom_implementation)
106 scm_app_impl = importlib.import_module(custom_implementation)
106 scm_app_impl = importlib.import_module(custom_implementation)
107 else:
107 else:
108 scm_app_impl = scm_app
108 scm_app_impl = scm_app
109 return scm_app_impl
109 return scm_app_impl
110
110
111 def _get_by_id(self, repo_name):
111 def _get_by_id(self, repo_name):
112 """
112 """
113 Gets a special pattern _<ID> from clone url and tries to replace it
113 Gets a special pattern _<ID> from clone url and tries to replace it
114 with a repository_name for support of _<ID> non changable urls
114 with a repository_name for support of _<ID> non changable urls
115
115
116 :param repo_name:
116 :param repo_name:
117 """
117 """
118
118
119 data = repo_name.split('/')
119 data = repo_name.split('/')
120 if len(data) >= 2:
120 if len(data) >= 2:
121 from rhodecode.model.repo import RepoModel
121 from rhodecode.model.repo import RepoModel
122 by_id_match = RepoModel().get_repo_by_id(repo_name)
122 by_id_match = RepoModel().get_repo_by_id(repo_name)
123 if by_id_match:
123 if by_id_match:
124 data[1] = by_id_match.repo_name
124 data[1] = by_id_match.repo_name
125
125
126 return safe_str('/'.join(data))
126 return safe_str('/'.join(data))
127
127
128 def _invalidate_cache(self, repo_name):
128 def _invalidate_cache(self, repo_name):
129 """
129 """
130 Set's cache for this repository for invalidation on next access
130 Set's cache for this repository for invalidation on next access
131
131
132 :param repo_name: full repo name, also a cache key
132 :param repo_name: full repo name, also a cache key
133 """
133 """
134 ScmModel().mark_for_invalidation(repo_name)
134 ScmModel().mark_for_invalidation(repo_name)
135
135
136 def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
136 def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
137 db_repo = Repository.get_by_repo_name(repo_name)
137 db_repo = Repository.get_by_repo_name(repo_name)
138 if not db_repo:
138 if not db_repo:
139 log.debug('Repository `%s` not found inside the database.',
139 log.debug('Repository `%s` not found inside the database.',
140 repo_name)
140 repo_name)
141 return False
141 return False
142
142
143 if db_repo.repo_type != scm_type:
143 if db_repo.repo_type != scm_type:
144 log.warning(
144 log.warning(
145 'Repository `%s` have incorrect scm_type, expected %s got %s',
145 'Repository `%s` have incorrect scm_type, expected %s got %s',
146 repo_name, db_repo.repo_type, scm_type)
146 repo_name, db_repo.repo_type, scm_type)
147 return False
147 return False
148
148
149 return is_valid_repo(repo_name, base_path, expect_scm=scm_type)
149 return is_valid_repo(repo_name, base_path, expect_scm=scm_type)
150
150
151 def valid_and_active_user(self, user):
151 def valid_and_active_user(self, user):
152 """
152 """
153 Checks if that user is not empty, and if it's actually object it checks
153 Checks if that user is not empty, and if it's actually object it checks
154 if he's active.
154 if he's active.
155
155
156 :param user: user object or None
156 :param user: user object or None
157 :return: boolean
157 :return: boolean
158 """
158 """
159 if user is None:
159 if user is None:
160 return False
160 return False
161
161
162 elif user.active:
162 elif user.active:
163 return True
163 return True
164
164
165 return False
165 return False
166
166
167 def _check_permission(self, action, user, repo_name, ip_addr=None):
167 def _check_permission(self, action, user, repo_name, ip_addr=None):
168 """
168 """
169 Checks permissions using action (push/pull) user and repository
169 Checks permissions using action (push/pull) user and repository
170 name
170 name
171
171
172 :param action: push or pull action
172 :param action: push or pull action
173 :param user: user instance
173 :param user: user instance
174 :param repo_name: repository name
174 :param repo_name: repository name
175 """
175 """
176 # check IP
176 # check IP
177 inherit = user.inherit_default_permissions
177 inherit = user.inherit_default_permissions
178 ip_allowed = AuthUser.check_ip_allowed(user.user_id, ip_addr,
178 ip_allowed = AuthUser.check_ip_allowed(user.user_id, ip_addr,
179 inherit_from_default=inherit)
179 inherit_from_default=inherit)
180 if ip_allowed:
180 if ip_allowed:
181 log.info('Access for IP:%s allowed', ip_addr)
181 log.info('Access for IP:%s allowed', ip_addr)
182 else:
182 else:
183 return False
183 return False
184
184
185 if action == 'push':
185 if action == 'push':
186 if not HasPermissionAnyMiddleware('repository.write',
186 if not HasPermissionAnyMiddleware('repository.write',
187 'repository.admin')(user,
187 'repository.admin')(user,
188 repo_name):
188 repo_name):
189 return False
189 return False
190
190
191 else:
191 else:
192 # any other action need at least read permission
192 # any other action need at least read permission
193 if not HasPermissionAnyMiddleware('repository.read',
193 if not HasPermissionAnyMiddleware('repository.read',
194 'repository.write',
194 'repository.write',
195 'repository.admin')(user,
195 'repository.admin')(user,
196 repo_name):
196 repo_name):
197 return False
197 return False
198
198
199 return True
199 return True
200
200
201 def _check_ssl(self, environ, start_response):
201 def _check_ssl(self, environ, start_response):
202 """
202 """
203 Checks the SSL check flag and returns False if SSL is not present
203 Checks the SSL check flag and returns False if SSL is not present
204 and required True otherwise
204 and required True otherwise
205 """
205 """
206 org_proto = environ['wsgi._org_proto']
206 org_proto = environ['wsgi._org_proto']
207 # check if we have SSL required ! if not it's a bad request !
207 # check if we have SSL required ! if not it's a bad request !
208 require_ssl = str2bool(
208 require_ssl = str2bool(
209 SettingsModel().get_ui_by_key('push_ssl').ui_value)
209 SettingsModel().get_ui_by_key('push_ssl').ui_value)
210 if require_ssl and org_proto == 'http':
210 if require_ssl and org_proto == 'http':
211 log.debug('proto is %s and SSL is required BAD REQUEST !',
211 log.debug('proto is %s and SSL is required BAD REQUEST !',
212 org_proto)
212 org_proto)
213 return False
213 return False
214 return True
214 return True
215
215
216 def __call__(self, environ, start_response):
216 def __call__(self, environ, start_response):
217 try:
217 try:
218 return self._handle_request(environ, start_response)
218 return self._handle_request(environ, start_response)
219 except Exception:
219 except Exception:
220 log.exception("Exception while handling request")
220 log.exception("Exception while handling request")
221 appenlight.track_exception(environ)
221 appenlight.track_exception(environ)
222 return HTTPInternalServerError()(environ, start_response)
222 return HTTPInternalServerError()(environ, start_response)
223 finally:
223 finally:
224 meta.Session.remove()
224 meta.Session.remove()
225
225
226 def _handle_request(self, environ, start_response):
226 def _handle_request(self, environ, start_response):
227
227
228 if not self._check_ssl(environ, start_response):
228 if not self._check_ssl(environ, start_response):
229 reason = ('SSL required, while RhodeCode was unable '
229 reason = ('SSL required, while RhodeCode was unable '
230 'to detect this as SSL request')
230 'to detect this as SSL request')
231 log.debug('User not allowed to proceed, %s', reason)
231 log.debug('User not allowed to proceed, %s', reason)
232 return HTTPNotAcceptable(reason)(environ, start_response)
232 return HTTPNotAcceptable(reason)(environ, start_response)
233
233
234 ip_addr = get_ip_addr(environ)
234 ip_addr = get_ip_addr(environ)
235 username = None
235 username = None
236
236
237 # skip passing error to error controller
237 # skip passing error to error controller
238 environ['pylons.status_code_redirect'] = True
238 environ['pylons.status_code_redirect'] = True
239
239
240 # ======================================================================
240 # ======================================================================
241 # EXTRACT REPOSITORY NAME FROM ENV
241 # EXTRACT REPOSITORY NAME FROM ENV
242 # ======================================================================
242 # ======================================================================
243 environ['PATH_INFO'] = self._get_by_id(environ['PATH_INFO'])
243 environ['PATH_INFO'] = self._get_by_id(environ['PATH_INFO'])
244 repo_name = self._get_repository_name(environ)
244 repo_name = self._get_repository_name(environ)
245 environ['REPO_NAME'] = repo_name
245 environ['REPO_NAME'] = repo_name
246 log.debug('Extracted repo name is %s', repo_name)
246 log.debug('Extracted repo name is %s', repo_name)
247
247
248 # check for type, presence in database and on filesystem
248 # check for type, presence in database and on filesystem
249 if not self.is_valid_and_existing_repo(
249 if not self.is_valid_and_existing_repo(
250 repo_name, self.basepath, self.SCM):
250 repo_name, self.basepath, self.SCM):
251 return HTTPNotFound()(environ, start_response)
251 return HTTPNotFound()(environ, start_response)
252
252
253 # ======================================================================
253 # ======================================================================
254 # GET ACTION PULL or PUSH
254 # GET ACTION PULL or PUSH
255 # ======================================================================
255 # ======================================================================
256 action = self._get_action(environ)
256 action = self._get_action(environ)
257
257
258 # ======================================================================
258 # ======================================================================
259 # CHECK ANONYMOUS PERMISSION
259 # CHECK ANONYMOUS PERMISSION
260 # ======================================================================
260 # ======================================================================
261 if action in ['pull', 'push']:
261 if action in ['pull', 'push']:
262 anonymous_user = User.get_default_user()
262 anonymous_user = User.get_default_user()
263 username = anonymous_user.username
263 username = anonymous_user.username
264 if anonymous_user.active:
264 if anonymous_user.active:
265 # ONLY check permissions if the user is activated
265 # ONLY check permissions if the user is activated
266 anonymous_perm = self._check_permission(
266 anonymous_perm = self._check_permission(
267 action, anonymous_user, repo_name, ip_addr)
267 action, anonymous_user, repo_name, ip_addr)
268 else:
268 else:
269 anonymous_perm = False
269 anonymous_perm = False
270
270
271 if not anonymous_user.active or not anonymous_perm:
271 if not anonymous_user.active or not anonymous_perm:
272 if not anonymous_user.active:
272 if not anonymous_user.active:
273 log.debug('Anonymous access is disabled, running '
273 log.debug('Anonymous access is disabled, running '
274 'authentication')
274 'authentication')
275
275
276 if not anonymous_perm:
276 if not anonymous_perm:
277 log.debug('Not enough credentials to access this '
277 log.debug('Not enough credentials to access this '
278 'repository as anonymous user')
278 'repository as anonymous user')
279
279
280 username = None
280 username = None
281 # ==============================================================
281 # ==============================================================
282 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
282 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
283 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
283 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
284 # ==============================================================
284 # ==============================================================
285
285
286 # try to auth based on environ, container auth methods
286 # try to auth based on environ, container auth methods
287 log.debug('Running PRE-AUTH for container based authentication')
287 log.debug('Running PRE-AUTH for container based authentication')
288 pre_auth = authenticate(
288 pre_auth = authenticate(
289 '', '', environ, VCS_TYPE, registry=self.registry)
289 '', '', environ, VCS_TYPE, registry=self.registry)
290 if pre_auth and pre_auth.get('username'):
290 if pre_auth and pre_auth.get('username'):
291 username = pre_auth['username']
291 username = pre_auth['username']
292 log.debug('PRE-AUTH got %s as username', username)
292 log.debug('PRE-AUTH got %s as username', username)
293
293
294 # If not authenticated by the container, running basic auth
294 # If not authenticated by the container, running basic auth
295 if not username:
295 if not username:
296 self.authenticate.realm = get_rhodecode_realm()
296 self.authenticate.realm = get_rhodecode_realm()
297
297
298 try:
298 try:
299 result = self.authenticate(environ)
299 result = self.authenticate(environ)
300 except (UserCreationError, NotAllowedToCreateUserError) as e:
300 except (UserCreationError, NotAllowedToCreateUserError) as e:
301 log.error(e)
301 log.error(e)
302 reason = safe_str(e)
302 reason = safe_str(e)
303 return HTTPNotAcceptable(reason)(environ, start_response)
303 return HTTPNotAcceptable(reason)(environ, start_response)
304
304
305 if isinstance(result, str):
305 if isinstance(result, str):
306 AUTH_TYPE.update(environ, 'basic')
306 AUTH_TYPE.update(environ, 'basic')
307 REMOTE_USER.update(environ, result)
307 REMOTE_USER.update(environ, result)
308 username = result
308 username = result
309 else:
309 else:
310 return result.wsgi_application(environ, start_response)
310 return result.wsgi_application(environ, start_response)
311
311
312 # ==============================================================
312 # ==============================================================
313 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
313 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
314 # ==============================================================
314 # ==============================================================
315 user = User.get_by_username(username)
315 user = User.get_by_username(username)
316 if not self.valid_and_active_user(user):
316 if not self.valid_and_active_user(user):
317 return HTTPForbidden()(environ, start_response)
317 return HTTPForbidden()(environ, start_response)
318 username = user.username
318 username = user.username
319 user.update_lastactivity()
319 user.update_lastactivity()
320 meta.Session().commit()
320 meta.Session().commit()
321
321
322 # check user attributes for password change flag
322 # check user attributes for password change flag
323 user_obj = user
323 user_obj = user
324 if user_obj and user_obj.username != User.DEFAULT_USER and \
324 if user_obj and user_obj.username != User.DEFAULT_USER and \
325 user_obj.user_data.get('force_password_change'):
325 user_obj.user_data.get('force_password_change'):
326 reason = 'password change required'
326 reason = 'password change required'
327 log.debug('User not allowed to authenticate, %s', reason)
327 log.debug('User not allowed to authenticate, %s', reason)
328 return HTTPNotAcceptable(reason)(environ, start_response)
328 return HTTPNotAcceptable(reason)(environ, start_response)
329
329
330 # check permissions for this repository
330 # check permissions for this repository
331 perm = self._check_permission(action, user, repo_name, ip_addr)
331 perm = self._check_permission(action, user, repo_name, ip_addr)
332 if not perm:
332 if not perm:
333 return HTTPForbidden()(environ, start_response)
333 return HTTPForbidden()(environ, start_response)
334
334
335 # extras are injected into UI object and later available
335 # extras are injected into UI object and later available
336 # in hooks executed by rhodecode
336 # in hooks executed by rhodecode
337 check_locking = _should_check_locking(environ.get('QUERY_STRING'))
337 check_locking = _should_check_locking(environ.get('QUERY_STRING'))
338 extras = vcs_operation_context(
338 extras = vcs_operation_context(
339 environ, repo_name=repo_name, username=username,
339 environ, repo_name=repo_name, username=username,
340 action=action, scm=self.SCM,
340 action=action, scm=self.SCM,
341 check_locking=check_locking)
341 check_locking=check_locking)
342
342
343 # ======================================================================
343 # ======================================================================
344 # REQUEST HANDLING
344 # REQUEST HANDLING
345 # ======================================================================
345 # ======================================================================
346 str_repo_name = safe_str(repo_name)
346 str_repo_name = safe_str(repo_name)
347 repo_path = os.path.join(safe_str(self.basepath), str_repo_name)
347 repo_path = os.path.join(safe_str(self.basepath), str_repo_name)
348 log.debug('Repository path is %s', repo_path)
348 log.debug('Repository path is %s', repo_path)
349
349
350 fix_PATH()
350 fix_PATH()
351
351
352 log.info(
352 log.info(
353 '%s action on %s repo "%s" by "%s" from %s',
353 '%s action on %s repo "%s" by "%s" from %s',
354 action, self.SCM, str_repo_name, safe_str(username), ip_addr)
354 action, self.SCM, str_repo_name, safe_str(username), ip_addr)
355
355
356 return self._generate_vcs_response(
356 return self._generate_vcs_response(
357 environ, start_response, repo_path, repo_name, extras, action)
357 environ, start_response, repo_path, repo_name, extras, action)
358
358
359 @initialize_generator
359 @initialize_generator
360 def _generate_vcs_response(
360 def _generate_vcs_response(
361 self, environ, start_response, repo_path, repo_name, extras,
361 self, environ, start_response, repo_path, repo_name, extras,
362 action):
362 action):
363 """
363 """
364 Returns a generator for the response content.
364 Returns a generator for the response content.
365
365
366 This method is implemented as a generator, so that it can trigger
366 This method is implemented as a generator, so that it can trigger
367 the cache validation after all content sent back to the client. It
367 the cache validation after all content sent back to the client. It
368 also handles the locking exceptions which will be triggered when
368 also handles the locking exceptions which will be triggered when
369 the first chunk is produced by the underlying WSGI application.
369 the first chunk is produced by the underlying WSGI application.
370 """
370 """
371 callback_daemon, extras = self._prepare_callback_daemon(extras)
371 callback_daemon, extras = self._prepare_callback_daemon(extras)
372 config = self._create_config(extras, repo_name)
372 config = self._create_config(extras, repo_name)
373 log.debug('HOOKS extras is %s', extras)
373 log.debug('HOOKS extras is %s', extras)
374 app = self._create_wsgi_app(repo_path, repo_name, config)
374 app = self._create_wsgi_app(repo_path, repo_name, config)
375
375
376 try:
376 try:
377 with callback_daemon:
377 with callback_daemon:
378 try:
378 try:
379 response = app(environ, start_response)
379 response = app(environ, start_response)
380 finally:
380 finally:
381 # This statement works together with the decorator
381 # This statement works together with the decorator
382 # "initialize_generator" above. The decorator ensures that
382 # "initialize_generator" above. The decorator ensures that
383 # we hit the first yield statement before the generator is
383 # we hit the first yield statement before the generator is
384 # returned back to the WSGI server. This is needed to
384 # returned back to the WSGI server. This is needed to
385 # ensure that the call to "app" above triggers the
385 # ensure that the call to "app" above triggers the
386 # needed callback to "start_response" before the
386 # needed callback to "start_response" before the
387 # generator is actually used.
387 # generator is actually used.
388 yield "__init__"
388 yield "__init__"
389
389
390 for chunk in response:
390 for chunk in response:
391 yield chunk
391 yield chunk
392 except Exception as exc:
392 except Exception as exc:
393 # TODO: johbo: Improve "translating" back the exception.
393 # TODO: johbo: Improve "translating" back the exception.
394 if getattr(exc, '_vcs_kind', None) == 'repo_locked':
394 if getattr(exc, '_vcs_kind', None) == 'repo_locked':
395 exc = HTTPLockedRC(*exc.args)
395 exc = HTTPLockedRC(*exc.args)
396 _code = rhodecode.CONFIG.get('lock_ret_code')
396 _code = rhodecode.CONFIG.get('lock_ret_code')
397 log.debug('Repository LOCKED ret code %s!', (_code,))
397 log.debug('Repository LOCKED ret code %s!', (_code,))
398 elif getattr(exc, '_vcs_kind', None) == 'requirement':
398 elif getattr(exc, '_vcs_kind', None) == 'requirement':
399 log.debug(
399 log.debug(
400 'Repository requires features unknown to this Mercurial')
400 'Repository requires features unknown to this Mercurial')
401 exc = HTTPRequirementError(*exc.args)
401 exc = HTTPRequirementError(*exc.args)
402 else:
402 else:
403 raise
403 raise
404
404
405 for chunk in exc(environ, start_response):
405 for chunk in exc(environ, start_response):
406 yield chunk
406 yield chunk
407 finally:
407 finally:
408 # invalidate cache on push
408 # invalidate cache on push
409 try:
409 if action == 'push':
410 if action == 'push':
410 self._invalidate_cache(repo_name)
411 self._invalidate_cache(repo_name)
412 finally:
413 meta.Session.remove()
411
414
412 def _get_repository_name(self, environ):
415 def _get_repository_name(self, environ):
413 """Get repository name out of the environmnent
416 """Get repository name out of the environmnent
414
417
415 :param environ: WSGI environment
418 :param environ: WSGI environment
416 """
419 """
417 raise NotImplementedError()
420 raise NotImplementedError()
418
421
419 def _get_action(self, environ):
422 def _get_action(self, environ):
420 """Map request commands into a pull or push command.
423 """Map request commands into a pull or push command.
421
424
422 :param environ: WSGI environment
425 :param environ: WSGI environment
423 """
426 """
424 raise NotImplementedError()
427 raise NotImplementedError()
425
428
426 def _create_wsgi_app(self, repo_path, repo_name, config):
429 def _create_wsgi_app(self, repo_path, repo_name, config):
427 """Return the WSGI app that will finally handle the request."""
430 """Return the WSGI app that will finally handle the request."""
428 raise NotImplementedError()
431 raise NotImplementedError()
429
432
430 def _create_config(self, extras, repo_name):
433 def _create_config(self, extras, repo_name):
431 """Create a Pyro safe config representation."""
434 """Create a Pyro safe config representation."""
432 raise NotImplementedError()
435 raise NotImplementedError()
433
436
434 def _prepare_callback_daemon(self, extras):
437 def _prepare_callback_daemon(self, extras):
435 return prepare_callback_daemon(
438 return prepare_callback_daemon(
436 extras, protocol=vcs_settings.HOOKS_PROTOCOL,
439 extras, protocol=vcs_settings.HOOKS_PROTOCOL,
437 use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS)
440 use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS)
438
441
439
442
440 def _should_check_locking(query_string):
443 def _should_check_locking(query_string):
441 # this is kind of hacky, but due to how mercurial handles client-server
444 # this is kind of hacky, but due to how mercurial handles client-server
442 # server see all operation on commit; bookmarks, phases and
445 # server see all operation on commit; bookmarks, phases and
443 # obsolescence marker in different transaction, we don't want to check
446 # obsolescence marker in different transaction, we don't want to check
444 # locking on those
447 # locking on those
445 return query_string not in ['cmd=listkeys']
448 return query_string not in ['cmd=listkeys']
@@ -1,86 +1,81 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2016 RhodeCode GmbH
3 # Copyright (C) 2010-2016 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21
21
22 import logging
22 import logging
23 import pylons
23 import pylons
24 import rhodecode
24 import rhodecode
25
25
26 from pylons.i18n.translation import _get_translator
26 from pylons.i18n.translation import _get_translator
27 from pylons.util import ContextObj
27 from pylons.util import ContextObj
28 from routes.util import URLGenerator
28 from routes.util import URLGenerator
29
29
30 from rhodecode.lib.base import attach_context_attributes, get_auth_user
30 from rhodecode.lib.base import attach_context_attributes, get_auth_user
31 from rhodecode.model import meta
31 from rhodecode.model import meta
32
32
33 log = logging.getLogger(__name__)
33 log = logging.getLogger(__name__)
34
34
35
35
36 def pylons_compatibility_tween_factory(handler, registry):
36 def pylons_compatibility_tween_factory(handler, registry):
37 def pylons_compatibility_tween(request):
37 def pylons_compatibility_tween(request):
38 """
38 """
39 While migrating from pylons to pyramid we need to call some pylons code
39 While migrating from pylons to pyramid we need to call some pylons code
40 from pyramid. For example while rendering an old template that uses the
40 from pyramid. For example while rendering an old template that uses the
41 'c' or 'h' objects. This tween sets up the needed pylons globals.
41 'c' or 'h' objects. This tween sets up the needed pylons globals.
42 """
42 """
43 try:
44 config = rhodecode.CONFIG
43 config = rhodecode.CONFIG
45 environ = request.environ
44 environ = request.environ
46 session = request.session
45 session = request.session
47 session_key = (config['pylons.environ_config']
46 session_key = (config['pylons.environ_config']
48 .get('session', 'beaker.session'))
47 .get('session', 'beaker.session'))
49
48
50 # Setup pylons globals.
49 # Setup pylons globals.
51 pylons.config._push_object(config)
50 pylons.config._push_object(config)
52 pylons.request._push_object(request)
51 pylons.request._push_object(request)
53 pylons.session._push_object(session)
52 pylons.session._push_object(session)
54 environ[session_key] = session
53 environ[session_key] = session
55 pylons.url._push_object(URLGenerator(config['routes.map'],
54 pylons.url._push_object(URLGenerator(config['routes.map'],
56 environ))
55 environ))
57
56
58 # TODO: Maybe we should use the language from pyramid.
57 # TODO: Maybe we should use the language from pyramid.
59 translator = _get_translator(config.get('lang'))
58 translator = _get_translator(config.get('lang'))
60 pylons.translator._push_object(translator)
59 pylons.translator._push_object(translator)
61
60
62 # Get the rhodecode auth user object and make it available.
61 # Get the rhodecode auth user object and make it available.
63 auth_user = get_auth_user(environ)
62 auth_user = get_auth_user(environ)
64 request.user = auth_user
63 request.user = auth_user
65 environ['rc_auth_user'] = auth_user
64 environ['rc_auth_user'] = auth_user
66
65
67 # Setup the pylons context object ('c')
66 # Setup the pylons context object ('c')
68 context = ContextObj()
67 context = ContextObj()
69 context.rhodecode_user = auth_user
68 context.rhodecode_user = auth_user
70 attach_context_attributes(context, request)
69 attach_context_attributes(context, request)
71 pylons.tmpl_context._push_object(context)
70 pylons.tmpl_context._push_object(context)
72 return handler(request)
71 return handler(request)
73 finally:
74 # Dispose current database session and rollback uncommitted
75 # transactions.
76 meta.Session.remove()
77
72
78 return pylons_compatibility_tween
73 return pylons_compatibility_tween
79
74
80
75
81 def includeme(config):
76 def includeme(config):
82 config.add_subscriber('rhodecode.subscribers.add_renderer_globals',
77 config.add_subscriber('rhodecode.subscribers.add_renderer_globals',
83 'pyramid.events.BeforeRender')
78 'pyramid.events.BeforeRender')
84 config.add_subscriber('rhodecode.subscribers.add_localizer',
79 config.add_subscriber('rhodecode.subscribers.add_localizer',
85 'pyramid.events.NewRequest')
80 'pyramid.events.NewRequest')
86 config.add_tween('rhodecode.tweens.pylons_compatibility_tween_factory')
81 config.add_tween('rhodecode.tweens.pylons_compatibility_tween_factory')
General Comments 0
You need to be logged in to leave comments. Login now