Implemented simple gist functionality ref #530.

- creation of public/private gists with given lifetime
- rhodecode-gist CLI for quick gist creation

rhodecode/bin/rhodecode_gist.py

@@ -0,0 +1,159 b''
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.bin.gist
+    ~~~~~~~~~~~~~~~~~~
+
+    Gist CLI client for RhodeCode
+
+    :created_on: May 9, 2013
+    :author: marcink
+    :copyright: (C) 2010-2013 Marcin Kuzminski <[e-mail]>
+    :license: GPLv3, see COPYING for more details.
+"""
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import with_statement
+import os
+import sys
+import stat
+import argparse
+import fileinput
+
+from rhodecode.bin.base import api_call, RcConf
+
+
+def argparser(argv):
+    usage = (
+      "rhodecode-gist [-h] [--format=FORMAT] [--apikey=APIKEY] [--apihost=APIHOST] "
+      "[--config=CONFIG] [--save-config] "
+      "[filename or stdin use - for terminal stdin ]\n"
+      "Create config file: rhodecode-gist --apikey=<key> --apihost=http://rhodecode.server --save-config"
+    )
+
+    parser = argparse.ArgumentParser(description='RhodeCode Gist cli',
+                                     usage=usage)
+
+    ## config
+    group = parser.add_argument_group('config')
+    group.add_argument('--apikey', help='api access key')
+    group.add_argument('--apihost', help='api host')
+    group.add_argument('--config', help='config file')
+    group.add_argument('--save-config', action='store_true',
+                       help='save the given config into a file')
+
+    group = parser.add_argument_group('GIST')
+    group.add_argument('-f', '--filename', help='set uploaded gist filename')
+    group.add_argument('-p', '--private', action='store_true',
+                       help='Create private Gist')
+    group.add_argument('-d', '--description', help='Gist description')
+    group.add_argument('-l', '--lifetime', metavar='MINUTES',
+                       help='Gist lifetime in minutes, -1 (Default) is forever')
+
+    args, other = parser.parse_known_args()
+    return parser, args, other
+
+
+def _run(argv):
+    conf = None
+    parser, args, other = argparser(argv)
+
+    api_credentials_given = (args.apikey and args.apihost)
+    if args.save_config:
+        if not api_credentials_given:
+            raise parser.error('--save-config requires --apikey and --apihost')
+        conf = RcConf(config_location=args.config,
+                      autocreate=True, config={'apikey': args.apikey,
+                                               'apihost': args.apihost})
+        sys.exit()
+
+    if not conf:
+        conf = RcConf(config_location=args.config, autoload=True)
+        if not conf:
+            if not api_credentials_given:
+                parser.error('Could not find config file and missing '
+                             '--apikey or --apihost in params')
+
+    apikey = args.apikey or conf['apikey']
+    host = args.apihost or conf['apihost']
+    DEFAULT_FILENAME = 'gistfile1.txt'
+    if other:
+        # skip multifiles for now
+        filename = other[0]
+        if filename == '-':
+            filename = DEFAULT_FILENAME
+            gist_content = ''
+            for line in fileinput.input():
+                gist_content += line
+        else:
+            with open(filename, 'rb') as f:
+                gist_content = f.read()
+
+    else:
+        filename = DEFAULT_FILENAME
+        gist_content = None
+        # little bit hacky but cross platform check where the
+        # stdin comes from we skip the terminal case it can be handled by '-'
+        mode = os.fstat(0).st_mode
+        if stat.S_ISFIFO(mode):
+            # "stdin is piped"
+            gist_content = sys.stdin.read()
+        elif stat.S_ISREG(mode):
+            # "stdin is redirected"
+            gist_content = sys.stdin.read()
+        else:
+            # "stdin is terminal"
+            pass
+
+    # make sure we don't upload binary stuff
+    if gist_content and '\0' in gist_content:
+        raise Exception('Error: binary files upload is not possible')
+
+    filename = args.filename or filename
+    if gist_content:
+        files = {
+            filename: {
+                'content': gist_content,
+                'lexer': None
+            }
+        }
+
+        margs = dict(
+            gist_lifetime=args.lifetime,
+            gist_description=args.description,
+            gist_type='private' if args.private else 'public',
+            files=files
+        )
+
+        api_call(apikey, host, 'json', 'create_gist', **margs)
+    return 0
+
+
+def main(argv=None):
+    """
+    Main execution function for cli
+
+    :param argv:
+    """
+    if argv is None:
+        argv = sys.argv
+
+    try:
+        return _run(argv)
+    except Exception, e:
+        print e
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

rhodecode/controllers/admin/gists.py

@@ -0,0 +1,180 b''
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.controllers.admin.gist
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    gist controller for RhodeCode
+
+    :created_on: May 9, 2013
+    :author: marcink
+    :copyright: (C) 2010-2013 Marcin Kuzminski <[e-mail]>
+    :license: GPLv3, see COPYING for more details.
+"""
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+import time
+import logging
+import traceback
+import formencode
+from formencode import htmlfill
+
+from pylons import request, tmpl_context as c, url
+from pylons.controllers.util import abort, redirect
+from pylons.i18n.translation import _
+
+from rhodecode.model.forms import GistForm
+from rhodecode.model.gist import GistModel
+from rhodecode.model.meta import Session
+from rhodecode.model.db import Gist
+from rhodecode.lib import helpers as h
+from rhodecode.lib.base import BaseController, render
+from rhodecode.lib.auth import LoginRequired, NotAnonymous
+from rhodecode.lib.utils2 import safe_str, safe_int, time_to_datetime
+from rhodecode.lib.helpers import Page
+from webob.exc import HTTPNotFound
+from sqlalchemy.sql.expression import or_
+from rhodecode.lib.vcs.exceptions import VCSError
+
+log = logging.getLogger(__name__)
+
+
+class GistsController(BaseController):
+    """REST Controller styled on the Atom Publishing Protocol"""
+
+    def __load_defaults(self):
+        c.lifetime_values = [
+            (str(-1), _('forever')),
+            (str(5), _('5 minutes')),
+            (str(60), _('1 hour')),
+            (str(60 * 24), _('1 day')),
+            (str(60 * 24 * 30), _('1 month')),
+        ]
+        c.lifetime_options = [(c.lifetime_values, _("Lifetime"))]
+
+    @LoginRequired()
+    def index(self, format='html'):
+        """GET /admin/gists: All items in the collection"""
+        # url('gists')
+        c.show_private = request.GET.get('private') and c.rhodecode_user.username != 'default'
+        gists = Gist().query()\
+            .filter(or_(Gist.gist_expires == -1, Gist.gist_expires >= time.time()))\
+            .order_by(Gist.created_on.desc())
+        if c.show_private:
+            c.gists = gists.filter(Gist.gist_type == Gist.GIST_PRIVATE)\
+                             .filter(Gist.gist_owner == c.rhodecode_user.user_id)
+        else:
+            c.gists = gists.filter(Gist.gist_type == Gist.GIST_PUBLIC)
+        p = safe_int(request.GET.get('page', 1), 1)
+        c.gists_pager = Page(c.gists, page=p, items_per_page=10)
+        return render('admin/gists/index.html')
+
+    @LoginRequired()
+    @NotAnonymous()
+    def create(self):
+        """POST /admin/gists: Create a new item"""
+        # url('gists')
+        self.__load_defaults()
+        gist_form = GistForm([x[0] for x in c.lifetime_values])()
+        try:
+            form_result = gist_form.to_python(dict(request.POST))
+            #TODO: multiple files support, from the form
+            nodes = {
+                form_result['filename'] or 'gistfile1.txt': {
+                    'content': form_result['content'],
+                    'lexer': None  # autodetect
+                }
+            }
+            _public = form_result['public']
+            gist_type = Gist.GIST_PUBLIC if _public else Gist.GIST_PRIVATE
+            gist = GistModel().create(
+                description=form_result['description'],
+                owner=c.rhodecode_user,
+                gist_mapping=nodes,
+                gist_type=gist_type,
+                lifetime=form_result['lifetime']
+            )
+            Session().commit()
+            new_gist_id = gist.gist_access_id
+        except formencode.Invalid, errors:
+            defaults = errors.value
+
+            return formencode.htmlfill.render(
+                render('admin/gists/new.html'),
+                defaults=defaults,
+                errors=errors.error_dict or {},
+                prefix_error=False,
+                encoding="UTF-8"
+            )
+
+        except Exception, e:
+            log.error(traceback.format_exc())
+            h.flash(_('Error occurred during gist creation'), category='error')
+            return redirect(url('new_gist'))
+        return redirect(url('gist', id=new_gist_id))
+
+    @LoginRequired()
+    @NotAnonymous()
+    def new(self, format='html'):
+        """GET /admin/gists/new: Form to create a new item"""
+        # url('new_gist')
+        self.__load_defaults()
+        return render('admin/gists/new.html')
+
+    @LoginRequired()
+    @NotAnonymous()
+    def update(self, id):
+        """PUT /admin/gists/id: Update an existing item"""
+        # Forms posted to this method should contain a hidden field:
+        #    <input type="hidden" name="_method" value="PUT" />
+        # Or using helpers:
+        #    h.form(url('gist', id=ID),
+        #           method='put')
+        # url('gist', id=ID)
+
+    @LoginRequired()
+    @NotAnonymous()
+    def delete(self, id):
+        """DELETE /admin/gists/id: Delete an existing item"""
+        # Forms posted to this method should contain a hidden field:
+        #    <input type="hidden" name="_method" value="DELETE" />
+        # Or using helpers:
+        #    h.form(url('gist', id=ID),
+        #           method='delete')
+        # url('gist', id=ID)
+
+    @LoginRequired()
+    def show(self, id, format='html'):
+        """GET /admin/gists/id: Show a specific item"""
+        # url('gist', id=ID)
+        gist_id = id
+        c.gist = Gist.get_or_404(gist_id)
+
+        #check if this gist is not expired
+        if c.gist.gist_expires != -1:
+            if time.time() > c.gist.gist_expires:
+                log.error('Gist expired at %s' %
+                          (time_to_datetime(c.gist.gist_expires)))
+                raise HTTPNotFound()
+        try:
+            c.file_changeset, c.files = GistModel().get_gist_files(gist_id)
+        except VCSError:
+            log.error(traceback.format_exc())
+            raise HTTPNotFound()
+
+        return render('admin/gists/show.html')
+
+    @LoginRequired()
+    @NotAnonymous()
+    def edit(self, id, format='html'):
+        """GET /admin/gists/id/edit: Form to edit an existing item"""
+        # url('edit_gist', id=ID)

rhodecode/model/gist.py

@@ -0,0 +1,161 b''
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.model.gist
+    ~~~~~~~~~~~~~~~~~~~~
+
+    gist model for RhodeCode
+
+    :created_on: May 9, 2013
+    :author: marcink
+    :copyright: (C) 2011-2013 Marcin Kuzminski <[e-mail]>
+    :license: GPLv3, see COPYING for more details.
+"""
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import time
+import logging
+import traceback
+import shutil
+
+from pylons.i18n.translation import _
+from rhodecode.lib.utils2 import safe_unicode, unique_id, safe_int, \
+    time_to_datetime, safe_str, AttributeDict
+from rhodecode.lib import helpers as h
+from rhodecode.model import BaseModel
+from rhodecode.model.db import Gist
+from rhodecode.model.repo import RepoModel
+from rhodecode.model.scm import ScmModel
+from rhodecode.lib.vcs import get_repo
+
+log = logging.getLogger(__name__)
+
+GIST_STORE_LOC = '.gist_store'
+
+
+class GistModel(BaseModel):
+
+    def _get_gist(self, gist):
+        """
+        Helper method to get gist by ID, or gist_access_id as a fallback
+
+        :param gist: GistID, gist_access_id, or Gist instance
+        """
+        return self._get_instance(Gist, gist,
+                                  callback=Gist.get_by_access_id)
+
+    def __delete_gist(self, gist):
+        """
+        removes gist from filesystem
+
+        :param gist: gist object
+        """
+        root_path = RepoModel().repos_path
+        rm_path = os.path.join(root_path, GIST_STORE_LOC, gist.gist_access_id)
+        log.info("Removing %s" % (rm_path))
+        shutil.rmtree(rm_path)
+
+    def get_gist_files(self, gist_access_id):
+        """
+        Get files for given gist
+
+        :param gist_access_id:
+        """
+        root_path = RepoModel().repos_path
+        r = get_repo(os.path.join(*map(safe_str,
+                                [root_path, GIST_STORE_LOC, gist_access_id])))
+        cs = r.get_changeset()
+        return (
+         cs, [n for n in cs.get_node('/')]
+        )
+
+    def create(self, description, owner, gist_mapping,
+               gist_type=Gist.GIST_PUBLIC, lifetime=-1):
+        """
+
+        :param description: description of the gist
+        :param owner: user who created this gist
+        :param gist_mapping: mapping {filename:{'content':content},...}
+        :param gist_type: type of gist private/public
+        :param lifetime: in minutes, -1 == forever
+        """
+        gist_id = safe_unicode(unique_id(20))
+        lifetime = safe_int(lifetime, -1)
+        gist_expires = time.time() + (lifetime * 60) if lifetime != -1 else -1
+        log.debug('set GIST expiration date to: %s'
+                  % (time_to_datetime(gist_expires)
+                   if gist_expires != -1 else 'forever'))
+        #create the Database version
+        gist = Gist()
+        gist.gist_description = description
+        gist.gist_access_id = gist_id
+        gist.gist_owner = owner.user_id
+        gist.gist_expires = gist_expires
+        gist.gist_type = safe_unicode(gist_type)
+        self.sa.add(gist)
+        self.sa.flush()
+        if gist_type == Gist.GIST_PUBLIC:
+            # use DB ID for easy to use GIST ID
+            gist_id = safe_unicode(gist.gist_id)
+            gist.gist_access_id = gist_id
+            self.sa.add(gist)
+
+        gist_repo_path = os.path.join(GIST_STORE_LOC, gist_id)
+        log.debug('Creating new %s GIST repo in %s' % (gist_type, gist_repo_path))
+        repo = RepoModel()._create_repo(repo_name=gist_repo_path, alias='hg',
+                                        parent=None)
+
+        processed_mapping = {}
+        for filename in gist_mapping:
+            content = gist_mapping[filename]['content']
+            #TODO: expand support for setting explicit lexers
+#             if lexer is None:
+#                 try:
+#                     lexer = pygments.lexers.guess_lexer_for_filename(filename,content)
+#                 except pygments.util.ClassNotFound:
+#                     lexer = 'text'
+            processed_mapping[filename] = {'content': content}
+
+        # now create single multifile commit
+        message = 'added file'
+        message += 's: ' if len(processed_mapping) > 1 else ': '
+        message += ', '.join([x for x in processed_mapping])
+
+        #fake RhodeCode Repository object
+        fake_repo = AttributeDict(dict(
+            repo_name=gist_repo_path,
+            scm_instance_no_cache=lambda: repo,
+        ))
+        ScmModel().create_nodes(
+            user=owner.user_id, repo=fake_repo,
+            message=message,
+            nodes=processed_mapping,
+            trigger_push_hook=False
+        )
+
+        return gist
+
+    def delete(self, gist, fs_remove=True):
+        gist = self._get_gist(gist)
+
+        try:
+            self.sa.delete(gist)
+            if fs_remove:
+                self.__delete_gist(gist)
+            else:
+                log.debug('skipping removal from filesystem')
+
+        except Exception:
+            log.error(traceback.format_exc())
+            raise

rhodecode/bin/base.py

@@ -1,148 +1,148 b''
 """
 Base utils for shell scripts
 """
 import os
 import sys
 import random
 import urllib2
 import pprint
 
 try:
     from rhodecode.lib.ext_json import json
 except ImportError:
     try:
         import simplejson as json
     except ImportError:
         import json
 
 CONFIG_NAME = '.rhodecode'
 FORMAT_PRETTY = 'pretty'
 FORMAT_JSON = 'json'
 
 
 def api_call(apikey, apihost, format, method=None, **kw):
     """
     Api_call wrapper for RhodeCode
 
     :param apikey:
     :param apihost:
     :param format: formatting, pretty means prints and pprint of json
      json returns unparsed json
     :param method:
     """
     def _build_data(random_id):
         """
         Builds API data with given random ID
 
         :param random_id:
-        :type random_id:
         """
         return {
             "id": random_id,
             "api_key": apikey,
             "method": method,
             "args": kw
         }
 
     if not method:
         raise Exception('please specify method name !')
     id_ = random.randrange(1, 9999)
     req = urllib2.Request('%s/_admin/api' % apihost,
                       data=json.dumps(_build_data(id_)),
                       headers={'content-type': 'text/plain'})
     if format == FORMAT_PRETTY:
         sys.stdout.write('calling %s to %s \n' % (req.get_data(), apihost))
     ret = urllib2.urlopen(req)
     raw_json = ret.read()
     json_data = json.loads(raw_json)
     id_ret = json_data['id']
     _formatted_json = pprint.pformat(json_data)
     if id_ret == id_:
         if format == FORMAT_JSON:
             sys.stdout.write(str(raw_json))
         else:
             sys.stdout.write('rhodecode returned:\n%s\n' % (_formatted_json))
 
     else:
         raise Exception('something went wrong. '
                         'ID mismatch got %s, expected %s | %s' % (
                                             id_ret, id_, _formatted_json))
 
 
 class RcConf(object):
     """
     RhodeCode config for API
 
     conf = RcConf()
     conf['key']
 
     """
 
     def __init__(self, config_location=None, autoload=True, autocreate=False,
                  config=None):
-        self._conf_name = CONFIG_NAME if not config_location else config_location
+        HOME = os.getenv('HOME', os.getenv('USERPROFILE')) or ''
+        HOME_CONF = os.path.abspath(os.path.join(HOME, CONFIG_NAME))
+        self._conf_name = HOME_CONF if not config_location else config_location
         self._conf = {}
         if autocreate:
             self.make_config(config)
         if autoload:
             self._conf = self.load_config()
 
     def __getitem__(self, key):
         return self._conf[key]
 
     def __nonzero__(self):
         if self._conf:
             return True
         return False
 
     def __eq__(self):
         return self._conf.__eq__()
 
     def __repr__(self):
         return 'RcConf<%s>' % self._conf.__repr__()
 
     def make_config(self, config):
         """
         Saves given config as a JSON dump in the _conf_name location
 
         :param config:
-        :type config:
         """
         update = False
         if os.path.exists(self._conf_name):
             update = True
         with open(self._conf_name, 'wb') as f:
             json.dump(config, f, indent=4)
 
         if update:
             sys.stdout.write('Updated config in %s\n' % self._conf_name)
         else:
             sys.stdout.write('Created new config in %s\n' % self._conf_name)
 
     def update_config(self, new_config):
         """
         Reads the JSON config updates it's values with new_config and
         saves it back as JSON dump
 
         :param new_config:
         """
         config = {}
         try:
             with open(self._conf_name, 'rb') as conf:
                 config = json.load(conf)
         except IOError, e:
             sys.stderr.write(str(e) + '\n')
 
         config.update(new_config)
         self.make_config(config)
 
     def load_config(self):
         """
         Loads config from file and returns loaded JSON object
         """
         try:
             with open(self._conf_name, 'rb') as conf:
                 return  json.load(conf)
         except IOError, e:
             #sys.stderr.write(str(e) + '\n')
             pass

rhodecode/bin/rhodecode_api.py

@@ -1,107 +1,106 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.bin.api
     ~~~~~~~~~~~~~~~~~
 
     Api CLI client for RhodeCode
 
     :created_on: Jun 3, 2012
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from __future__ import with_statement
 import sys
 import argparse
 
 from rhodecode.bin.base import api_call, RcConf, FORMAT_JSON, FORMAT_PRETTY
 
 
 def argparser(argv):
     usage = (
       "rhodecode-api [-h] [--format=FORMAT] [--apikey=APIKEY] [--apihost=APIHOST] "
       "[--config=CONFIG] [--save-config] "
       "METHOD <key:val> <key2:val> ...\n"
       "Create config file: rhodecode-gist --apikey=<key> --apihost=http://rhodecode.server --save-config"
     )
 
     parser = argparse.ArgumentParser(description='RhodeCode API cli',
                                      usage=usage)
 
     ## config
     group = parser.add_argument_group('config')
     group.add_argument('--apikey', help='api access key')
     group.add_argument('--apihost', help='api host')
     group.add_argument('--config', help='config file')
     group.add_argument('--save-config', action='store_true', help='save the given config into a file')
 
     group = parser.add_argument_group('API')
     group.add_argument('method', metavar='METHOD', nargs='?', type=str, default=None,
             help='API method name to call followed by key:value attributes',
     )
     group.add_argument('--format', dest='format', type=str,
             help='output format default: `pretty` can '
                  'be also `%s`' % FORMAT_JSON,
             default=FORMAT_PRETTY
     )
     args, other = parser.parse_known_args()
     return parser, args, other
 
 
 def main(argv=None):
     """
     Main execution function for cli
 
     :param argv:
-    :type argv:
     """
     if argv is None:
         argv = sys.argv
 
     conf = None
     parser, args, other = argparser(argv)
 
     api_credentials_given = (args.apikey and args.apihost)
     if args.save_config:
         if not api_credentials_given:
             raise parser.error('--save-config requires --apikey and --apihost')
         conf = RcConf(config_location=args.config,
                       autocreate=True, config={'apikey': args.apikey,
                                                'apihost': args.apihost})
         sys.exit()
 
     if not conf:
         conf = RcConf(config_location=args.config, autoload=True)
         if not conf:
             if not api_credentials_given:
                 parser.error('Could not find config file and missing '
                              '--apikey or --apihost in params')
 
     apikey = args.apikey or conf['apikey']
     host = args.apihost or conf['apihost']
     method = args.method
 
     try:
         margs = dict(map(lambda s: s.split(':', 1), other))
     except Exception:
         sys.stderr.write('Error parsing arguments \n')
         sys.exit()
 
     api_call(apikey, host, args.format, method, **margs)
     return 0
 
 if __name__ == '__main__':
     sys.exit(main(sys.argv))

rhodecode/config/routing.py

@@ -1,680 +1,683 b''
 """
 Routes configuration
 
 The more specific and detailed routes should be defined first so they
 may take precedent over the more generic routes. For more information
 refer to the routes manual at http://routes.groovie.org/docs/
 """
 from __future__ import with_statement
 from routes import Mapper
 
 # prefix for non repository related links needs to be prefixed with `/`
 ADMIN_PREFIX = '/_admin'
 
 
 def make_map(config):
     """Create, configure and return the routes Mapper"""
     rmap = Mapper(directory=config['pylons.paths']['controllers'],
                  always_scan=config['debug'])
     rmap.minimization = False
     rmap.explicit = False
 
     from rhodecode.lib.utils import is_valid_repo
     from rhodecode.lib.utils import is_valid_repos_group
 
     def check_repo(environ, match_dict):
         """
         check for valid repository for proper 404 handling
 
         :param environ:
         :param match_dict:
         """
         from rhodecode.model.db import Repository
         repo_name = match_dict.get('repo_name')
 
         if match_dict.get('f_path'):
             #fix for multiple initial slashes that causes errors
             match_dict['f_path'] = match_dict['f_path'].lstrip('/')
 
         try:
             by_id = repo_name.split('_')
             if len(by_id) == 2 and by_id[1].isdigit() and by_id[0] == '':
                 repo_name = Repository.get(by_id[1]).repo_name
                 match_dict['repo_name'] = repo_name
         except Exception:
             pass
 
         return is_valid_repo(repo_name, config['base_path'])
 
     def check_group(environ, match_dict):
         """
         check for valid repository group for proper 404 handling
 
         :param environ:
         :param match_dict:
         """
         repos_group_name = match_dict.get('group_name')
         return is_valid_repos_group(repos_group_name, config['base_path'])
 
     def check_group_skip_path(environ, match_dict):
         """
         check for valid repository group for proper 404 handling, but skips
         verification of existing path
 
         :param environ:
         :param match_dict:
         """
         repos_group_name = match_dict.get('group_name')
         return is_valid_repos_group(repos_group_name, config['base_path'],
                                     skip_path_check=True)
 
     def check_user_group(environ, match_dict):
         """
         check for valid user group for proper 404 handling
 
         :param environ:
         :param match_dict:
         """
         return True
 
     def check_int(environ, match_dict):
         return match_dict.get('id').isdigit()
 
     # The ErrorController route (handles 404/500 error pages); it should
     # likely stay at the top, ensuring it can always be resolved
     rmap.connect('/error/{action}', controller='error')
     rmap.connect('/error/{action}/{id}', controller='error')
 
     #==========================================================================
     # CUSTOM ROUTES HERE
     #==========================================================================
 
     #MAIN PAGE
     rmap.connect('home', '/', controller='home', action='index')
     rmap.connect('repo_switcher', '/repos', controller='home',
                  action='repo_switcher')
     rmap.connect('branch_tag_switcher', '/branches-tags/{repo_name:.*?}',
                  controller='home', action='branch_tag_switcher')
     rmap.connect('bugtracker',
                  "http://bitbucket.org/marcinkuzminski/rhodecode/issues",
                  _static=True)
     rmap.connect('rst_help',
                  "http://docutils.sourceforge.net/docs/user/rst/quickref.html",
                  _static=True)
     rmap.connect('rhodecode_official', "http://rhodecode.org", _static=True)
 
     #ADMIN REPOSITORY REST ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/repos') as m:
         m.connect("repos", "/repos",
              action="create", conditions=dict(method=["POST"]))
         m.connect("repos", "/repos",
              action="index", conditions=dict(method=["GET"]))
         m.connect("formatted_repos", "/repos.{format}",
              action="index",
             conditions=dict(method=["GET"]))
         m.connect("new_repo", "/create_repository",
                   action="create_repository", conditions=dict(method=["GET"]))
         m.connect("/repos/{repo_name:.*?}",
              action="update", conditions=dict(method=["PUT"],
                                               function=check_repo))
         m.connect("/repos/{repo_name:.*?}",
              action="delete", conditions=dict(method=["DELETE"],
                                               function=check_repo))
         m.connect("formatted_edit_repo", "/repos/{repo_name:.*?}.{format}/edit",
              action="edit", conditions=dict(method=["GET"],
                                             function=check_repo))
         m.connect("repo", "/repos/{repo_name:.*?}",
              action="show", conditions=dict(method=["GET"],
                                             function=check_repo))
         m.connect("formatted_repo", "/repos/{repo_name:.*?}.{format}",
              action="show", conditions=dict(method=["GET"],
                                             function=check_repo))
         #add repo perm member
         m.connect('set_repo_perm_member',
                   "/repos/{repo_name:.*?}/grant_perm",
                   action="set_repo_perm_member",
                   conditions=dict(method=["POST"], function=check_repo))
 
         #ajax delete repo perm user
         m.connect('delete_repo_perm_member',
                   "/repos/{repo_name:.*?}/revoke_perm",
                   action="delete_repo_perm_member",
                   conditions=dict(method=["DELETE"], function=check_repo))
 
         #settings actions
         m.connect('repo_stats', "/repos_stats/{repo_name:.*?}",
                   action="repo_stats", conditions=dict(method=["DELETE"],
                                                        function=check_repo))
         m.connect('repo_cache', "/repos_cache/{repo_name:.*?}",
                   action="repo_cache", conditions=dict(method=["DELETE"],
                                                        function=check_repo))
         m.connect('repo_public_journal', "/repos_public_journal/{repo_name:.*?}",
                   action="repo_public_journal", conditions=dict(method=["PUT"],
                                                         function=check_repo))
         m.connect('repo_pull', "/repo_pull/{repo_name:.*?}",
                   action="repo_pull", conditions=dict(method=["PUT"],
                                                       function=check_repo))
         m.connect('repo_as_fork', "/repo_as_fork/{repo_name:.*?}",
                   action="repo_as_fork", conditions=dict(method=["PUT"],
                                                       function=check_repo))
         m.connect('repo_locking', "/repo_locking/{repo_name:.*?}",
                   action="repo_locking", conditions=dict(method=["PUT"],
                                                       function=check_repo))
         m.connect('toggle_locking', "/locking_toggle/{repo_name:.*?}",
                   action="toggle_locking", conditions=dict(method=["GET"],
                                                       function=check_repo))
 
         #repo fields
         m.connect('create_repo_fields', "/repo_fields/{repo_name:.*?}/new",
                   action="create_repo_field", conditions=dict(method=["PUT"],
                                                       function=check_repo))
 
         m.connect('delete_repo_fields', "/repo_fields/{repo_name:.*?}/{field_id}",
                   action="delete_repo_field", conditions=dict(method=["DELETE"],
                                                       function=check_repo))
 
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/repos_groups') as m:
         m.connect("repos_groups", "/repos_groups",
                   action="create", conditions=dict(method=["POST"]))
         m.connect("repos_groups", "/repos_groups",
                   action="index", conditions=dict(method=["GET"]))
         m.connect("formatted_repos_groups", "/repos_groups.{format}",
                   action="index", conditions=dict(method=["GET"]))
         m.connect("new_repos_group", "/repos_groups/new",
                   action="new", conditions=dict(method=["GET"]))
         m.connect("formatted_new_repos_group", "/repos_groups/new.{format}",
                   action="new", conditions=dict(method=["GET"]))
         m.connect("update_repos_group", "/repos_groups/{group_name:.*?}",
                   action="update", conditions=dict(method=["PUT"],
                                                    function=check_group))
         #add repo group perm member
         m.connect('set_repo_group_perm_member',
                   "/repos_groups/{group_name:.*?}/grant_perm",
                   action="set_repo_group_perm_member",
                   conditions=dict(method=["POST"], function=check_group))
 
         #ajax delete repo group perm
         m.connect('delete_repo_group_perm_member',
                   "/repos_groups/{group_name:.*?}/revoke_perm",
                   action="delete_repo_group_perm_member",
                   conditions=dict(method=["DELETE"], function=check_group))
 
         m.connect("delete_repos_group", "/repos_groups/{group_name:.*?}",
                   action="delete", conditions=dict(method=["DELETE"],
                                                    function=check_group_skip_path))
         m.connect("edit_repos_group", "/repos_groups/{group_name:.*?}/edit",
                   action="edit", conditions=dict(method=["GET"],
                                                  function=check_group))
         m.connect("formatted_edit_repos_group",
                   "/repos_groups/{group_name:.*?}.{format}/edit",
                   action="edit", conditions=dict(method=["GET"],
                                                  function=check_group))
         m.connect("repos_group", "/repos_groups/{group_name:.*?}",
                   action="show", conditions=dict(method=["GET"],
                                                  function=check_group))
         m.connect("formatted_repos_group", "/repos_groups/{group_name:.*?}.{format}",
                   action="show", conditions=dict(method=["GET"],
                                                  function=check_group))
 
     #ADMIN USER REST ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/users') as m:
         m.connect("users", "/users",
                   action="create", conditions=dict(method=["POST"]))
         m.connect("users", "/users",
                   action="index", conditions=dict(method=["GET"]))
         m.connect("formatted_users", "/users.{format}",
                   action="index", conditions=dict(method=["GET"]))
         m.connect("new_user", "/users/new",
                   action="new", conditions=dict(method=["GET"]))
         m.connect("formatted_new_user", "/users/new.{format}",
                   action="new", conditions=dict(method=["GET"]))
         m.connect("update_user", "/users/{id}",
                   action="update", conditions=dict(method=["PUT"]))
         m.connect("delete_user", "/users/{id}",
                   action="delete", conditions=dict(method=["DELETE"]))
         m.connect("edit_user", "/users/{id}/edit",
                   action="edit", conditions=dict(method=["GET"]))
         m.connect("formatted_edit_user",
                   "/users/{id}.{format}/edit",
                   action="edit", conditions=dict(method=["GET"]))
         m.connect("user", "/users/{id}",
                   action="show", conditions=dict(method=["GET"]))
         m.connect("formatted_user", "/users/{id}.{format}",
                   action="show", conditions=dict(method=["GET"]))
 
         #EXTRAS USER ROUTES
         m.connect("user_perm", "/users_perm/{id}",
                   action="update_perm", conditions=dict(method=["PUT"]))
         m.connect("user_emails", "/users_emails/{id}",
                   action="add_email", conditions=dict(method=["PUT"]))
         m.connect("user_emails_delete", "/users_emails/{id}",
                   action="delete_email", conditions=dict(method=["DELETE"]))
         m.connect("user_ips", "/users_ips/{id}",
                   action="add_ip", conditions=dict(method=["PUT"]))
         m.connect("user_ips_delete", "/users_ips/{id}",
                   action="delete_ip", conditions=dict(method=["DELETE"]))
 
     #ADMIN USER GROUPS REST ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/users_groups') as m:
         m.connect("users_groups", "/users_groups",
                   action="create", conditions=dict(method=["POST"]))
         m.connect("users_groups", "/users_groups",
                   action="index", conditions=dict(method=["GET"]))
         m.connect("formatted_users_groups", "/users_groups.{format}",
                   action="index", conditions=dict(method=["GET"]))
         m.connect("new_users_group", "/users_groups/new",
                   action="new", conditions=dict(method=["GET"]))
         m.connect("formatted_new_users_group", "/users_groups/new.{format}",
                   action="new", conditions=dict(method=["GET"]))
         m.connect("update_users_group", "/users_groups/{id}",
                   action="update", conditions=dict(method=["PUT"]))
         m.connect("delete_users_group", "/users_groups/{id}",
                   action="delete", conditions=dict(method=["DELETE"]))
         m.connect("edit_users_group", "/users_groups/{id}/edit",
                   action="edit", conditions=dict(method=["GET"]),
                   function=check_user_group)
         m.connect("formatted_edit_users_group",
                   "/users_groups/{id}.{format}/edit",
                   action="edit", conditions=dict(method=["GET"]))
         m.connect("users_group", "/users_groups/{id}",
                   action="show", conditions=dict(method=["GET"]))
         m.connect("formatted_users_group", "/users_groups/{id}.{format}",
                   action="show", conditions=dict(method=["GET"]))
 
         #EXTRAS USER ROUTES
         # update
         m.connect("users_group_perm", "/users_groups/{id}/update_global_perm",
                   action="update_perm", conditions=dict(method=["PUT"]))
 
         #add user group perm member
         m.connect('set_user_group_perm_member', "/users_groups/{id}/grant_perm",
              action="set_user_group_perm_member",
              conditions=dict(method=["POST"]))
 
         #ajax delete user group perm
         m.connect('delete_user_group_perm_member', "/users_groups/{id}/revoke_perm",
              action="delete_user_group_perm_member",
              conditions=dict(method=["DELETE"]))
 
     #ADMIN GROUP REST ROUTES
     rmap.resource('group', 'groups',
                   controller='admin/groups', path_prefix=ADMIN_PREFIX)
 
     #ADMIN PERMISSIONS REST ROUTES
     rmap.resource('permission', 'permissions',
                   controller='admin/permissions', path_prefix=ADMIN_PREFIX)
 
     #ADMIN DEFAULTS REST ROUTES
     rmap.resource('default', 'defaults',
                   controller='admin/defaults', path_prefix=ADMIN_PREFIX)
 
     ##ADMIN LDAP SETTINGS
     rmap.connect('ldap_settings', '%s/ldap' % ADMIN_PREFIX,
                  controller='admin/ldap_settings', action='ldap_settings',
                  conditions=dict(method=["POST"]))
 
     rmap.connect('ldap_home', '%s/ldap' % ADMIN_PREFIX,
                  controller='admin/ldap_settings')
 
     #ADMIN SETTINGS REST ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/settings') as m:
         m.connect("admin_settings", "/settings",
                   action="create", conditions=dict(method=["POST"]))
         m.connect("admin_settings", "/settings",
                   action="index", conditions=dict(method=["GET"]))
         m.connect("formatted_admin_settings", "/settings.{format}",
                   action="index", conditions=dict(method=["GET"]))
         m.connect("admin_new_setting", "/settings/new",
                   action="new", conditions=dict(method=["GET"]))
         m.connect("formatted_admin_new_setting", "/settings/new.{format}",
                   action="new", conditions=dict(method=["GET"]))
         m.connect("/settings/{setting_id}",
                   action="update", conditions=dict(method=["PUT"]))
         m.connect("/settings/{setting_id}",
                   action="delete", conditions=dict(method=["DELETE"]))
         m.connect("admin_edit_setting", "/settings/{setting_id}/edit",
                   action="edit", conditions=dict(method=["GET"]))
         m.connect("formatted_admin_edit_setting",
                   "/settings/{setting_id}.{format}/edit",
                   action="edit", conditions=dict(method=["GET"]))
         m.connect("admin_setting", "/settings/{setting_id}",
                   action="show", conditions=dict(method=["GET"]))
         m.connect("formatted_admin_setting", "/settings/{setting_id}.{format}",
                   action="show", conditions=dict(method=["GET"]))
         m.connect("admin_settings_my_account", "/my_account",
                   action="my_account", conditions=dict(method=["GET"]))
         m.connect("admin_settings_my_account_update", "/my_account_update",
                   action="my_account_update", conditions=dict(method=["PUT"]))
         m.connect("admin_settings_my_repos", "/my_account/repos",
                   action="my_account_my_repos", conditions=dict(method=["GET"]))
         m.connect("admin_settings_my_pullrequests", "/my_account/pull_requests",
                   action="my_account_my_pullrequests", conditions=dict(method=["GET"]))
 
     #NOTIFICATION REST ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/notifications') as m:
         m.connect("notifications", "/notifications",
                   action="create", conditions=dict(method=["POST"]))
         m.connect("notifications", "/notifications",
                   action="index", conditions=dict(method=["GET"]))
         m.connect("notifications_mark_all_read", "/notifications/mark_all_read",
                   action="mark_all_read", conditions=dict(method=["GET"]))
         m.connect("formatted_notifications", "/notifications.{format}",
                   action="index", conditions=dict(method=["GET"]))
         m.connect("new_notification", "/notifications/new",
                   action="new", conditions=dict(method=["GET"]))
         m.connect("formatted_new_notification", "/notifications/new.{format}",
                   action="new", conditions=dict(method=["GET"]))
         m.connect("/notification/{notification_id}",
                   action="update", conditions=dict(method=["PUT"]))
         m.connect("/notification/{notification_id}",
                   action="delete", conditions=dict(method=["DELETE"]))
         m.connect("edit_notification", "/notification/{notification_id}/edit",
                   action="edit", conditions=dict(method=["GET"]))
         m.connect("formatted_edit_notification",
                   "/notification/{notification_id}.{format}/edit",
                   action="edit", conditions=dict(method=["GET"]))
         m.connect("notification", "/notification/{notification_id}",
                   action="show", conditions=dict(method=["GET"]))
         m.connect("formatted_notification", "/notifications/{notification_id}.{format}",
                   action="show", conditions=dict(method=["GET"]))
 
     #ADMIN MAIN PAGES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/admin') as m:
         m.connect('admin_home', '', action='index')
         m.connect('admin_add_repo', '/add_repo/{new_repo:[a-z0-9\. _-]*}',
                   action='add_repo')
 
+    #ADMIN GIST
+    rmap.resource('gist', 'gists', controller='admin/gists',
+        path_prefix=ADMIN_PREFIX)
     #==========================================================================
     # API V2
     #==========================================================================
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='api/api') as m:
         m.connect('api', '/api')
 
     #USER JOURNAL
     rmap.connect('journal', '%s/journal' % ADMIN_PREFIX,
                  controller='journal', action='index')
     rmap.connect('journal_rss', '%s/journal/rss' % ADMIN_PREFIX,
                  controller='journal', action='journal_rss')
     rmap.connect('journal_atom', '%s/journal/atom' % ADMIN_PREFIX,
                  controller='journal', action='journal_atom')
 
     rmap.connect('public_journal', '%s/public_journal' % ADMIN_PREFIX,
                  controller='journal', action="public_journal")
 
     rmap.connect('public_journal_rss', '%s/public_journal/rss' % ADMIN_PREFIX,
                  controller='journal', action="public_journal_rss")
 
     rmap.connect('public_journal_rss_old', '%s/public_journal_rss' % ADMIN_PREFIX,
                  controller='journal', action="public_journal_rss")
 
     rmap.connect('public_journal_atom',
                  '%s/public_journal/atom' % ADMIN_PREFIX, controller='journal',
                  action="public_journal_atom")
 
     rmap.connect('public_journal_atom_old',
                  '%s/public_journal_atom' % ADMIN_PREFIX, controller='journal',
                  action="public_journal_atom")
 
     rmap.connect('toggle_following', '%s/toggle_following' % ADMIN_PREFIX,
                  controller='journal', action='toggle_following',
                  conditions=dict(method=["POST"]))
 
     #SEARCH
     rmap.connect('search', '%s/search' % ADMIN_PREFIX, controller='search',)
     rmap.connect('search_repo_admin', '%s/search/{repo_name:.*}' % ADMIN_PREFIX,
                  controller='search',
                  conditions=dict(function=check_repo))
     rmap.connect('search_repo', '/{repo_name:.*?}/search',
                  controller='search',
                  conditions=dict(function=check_repo),
                  )
 
     #LOGIN/LOGOUT/REGISTER/SIGN IN
     rmap.connect('login_home', '%s/login' % ADMIN_PREFIX, controller='login')
     rmap.connect('logout_home', '%s/logout' % ADMIN_PREFIX, controller='login',
                  action='logout')
 
     rmap.connect('register', '%s/register' % ADMIN_PREFIX, controller='login',
                  action='register')
 
     rmap.connect('reset_password', '%s/password_reset' % ADMIN_PREFIX,
                  controller='login', action='password_reset')
 
     rmap.connect('reset_password_confirmation',
                  '%s/password_reset_confirmation' % ADMIN_PREFIX,
                  controller='login', action='password_reset_confirmation')
 
     #FEEDS
     rmap.connect('rss_feed_home', '/{repo_name:.*?}/feed/rss',
                 controller='feed', action='rss',
                 conditions=dict(function=check_repo))
 
     rmap.connect('atom_feed_home', '/{repo_name:.*?}/feed/atom',
                 controller='feed', action='atom',
                 conditions=dict(function=check_repo))
 
     #==========================================================================
     # REPOSITORY ROUTES
     #==========================================================================
     rmap.connect('summary_home', '/{repo_name:.*?}',
                 controller='summary',
                 conditions=dict(function=check_repo))
 
     rmap.connect('repo_size', '/{repo_name:.*?}/repo_size',
                 controller='summary', action='repo_size',
                 conditions=dict(function=check_repo))
 
     rmap.connect('repos_group_home', '/{group_name:.*}',
                 controller='admin/repos_groups', action="show_by_name",
                 conditions=dict(function=check_group))
 
     rmap.connect('changeset_home', '/{repo_name:.*?}/changeset/{revision}',
                 controller='changeset', revision='tip',
                 conditions=dict(function=check_repo))
 
     # no longer user, but kept for routes to work
     rmap.connect("_edit_repo", "/{repo_name:.*?}/edit",
                  controller='admin/repos', action="edit",
                  conditions=dict(method=["GET"], function=check_repo)
                  )
 
     rmap.connect("edit_repo", "/{repo_name:.*?}/settings",
                  controller='admin/repos', action="edit",
                  conditions=dict(method=["GET"], function=check_repo)
                  )
 
     #still working url for backward compat.
     rmap.connect('raw_changeset_home_depraced',
                  '/{repo_name:.*?}/raw-changeset/{revision}',
                  controller='changeset', action='changeset_raw',
                  revision='tip', conditions=dict(function=check_repo))
 
     ## new URLs
     rmap.connect('changeset_raw_home',
                  '/{repo_name:.*?}/changeset-diff/{revision}',
                  controller='changeset', action='changeset_raw',
                  revision='tip', conditions=dict(function=check_repo))
 
     rmap.connect('changeset_patch_home',
                  '/{repo_name:.*?}/changeset-patch/{revision}',
                  controller='changeset', action='changeset_patch',
                  revision='tip', conditions=dict(function=check_repo))
 
     rmap.connect('changeset_download_home',
                  '/{repo_name:.*?}/changeset-download/{revision}',
                  controller='changeset', action='changeset_download',
                  revision='tip', conditions=dict(function=check_repo))
 
     rmap.connect('changeset_comment',
                  '/{repo_name:.*?}/changeset/{revision}/comment',
                 controller='changeset', revision='tip', action='comment',
                 conditions=dict(function=check_repo))
 
     rmap.connect('changeset_comment_preview',
                  '/{repo_name:.*?}/changeset/comment/preview',
                 controller='changeset', action='preview_comment',
                 conditions=dict(function=check_repo, method=["POST"]))
 
     rmap.connect('changeset_comment_delete',
                  '/{repo_name:.*?}/changeset/comment/{comment_id}/delete',
                 controller='changeset', action='delete_comment',
                 conditions=dict(function=check_repo, method=["DELETE"]))
 
     rmap.connect('changeset_info', '/changeset_info/{repo_name:.*?}/{revision}',
                  controller='changeset', action='changeset_info')
 
     rmap.connect('compare_url',
                  '/{repo_name:.*?}/compare/{org_ref_type}@{org_ref:.*?}...{other_ref_type}@{other_ref:.*?}',
                  controller='compare', action='index',
                  conditions=dict(function=check_repo),
                  requirements=dict(
                             org_ref_type='(branch|book|tag|rev|__other_ref_type__)',
                             other_ref_type='(branch|book|tag|rev|__org_ref_type__)')
                  )
 
     rmap.connect('pullrequest_home',
                  '/{repo_name:.*?}/pull-request/new', controller='pullrequests',
                  action='index', conditions=dict(function=check_repo,
                                                  method=["GET"]))
 
     rmap.connect('pullrequest',
                  '/{repo_name:.*?}/pull-request/new', controller='pullrequests',
                  action='create', conditions=dict(function=check_repo,
                                                   method=["POST"]))
 
     rmap.connect('pullrequest_show',
                  '/{repo_name:.*?}/pull-request/{pull_request_id}',
                  controller='pullrequests',
                  action='show', conditions=dict(function=check_repo,
                                                 method=["GET"]))
     rmap.connect('pullrequest_update',
                  '/{repo_name:.*?}/pull-request/{pull_request_id}',
                  controller='pullrequests',
                  action='update', conditions=dict(function=check_repo,
                                                 method=["PUT"]))
     rmap.connect('pullrequest_delete',
                  '/{repo_name:.*?}/pull-request/{pull_request_id}',
                  controller='pullrequests',
                  action='delete', conditions=dict(function=check_repo,
                                                 method=["DELETE"]))
 
     rmap.connect('pullrequest_show_all',
                  '/{repo_name:.*?}/pull-request',
                  controller='pullrequests',
                  action='show_all', conditions=dict(function=check_repo,
                                                 method=["GET"]))
 
     rmap.connect('pullrequest_comment',
                  '/{repo_name:.*?}/pull-request-comment/{pull_request_id}',
                  controller='pullrequests',
                  action='comment', conditions=dict(function=check_repo,
                                                 method=["POST"]))
 
     rmap.connect('pullrequest_comment_delete',
                  '/{repo_name:.*?}/pull-request-comment/{comment_id}/delete',
                 controller='pullrequests', action='delete_comment',
                 conditions=dict(function=check_repo, method=["DELETE"]))
 
     rmap.connect('summary_home_summary', '/{repo_name:.*?}/summary',
                 controller='summary', conditions=dict(function=check_repo))
 
     rmap.connect('branches_home', '/{repo_name:.*?}/branches',
                 controller='branches', conditions=dict(function=check_repo))
 
     rmap.connect('tags_home', '/{repo_name:.*?}/tags',
                 controller='tags', conditions=dict(function=check_repo))
 
     rmap.connect('bookmarks_home', '/{repo_name:.*?}/bookmarks',
                 controller='bookmarks', conditions=dict(function=check_repo))
 
     rmap.connect('changelog_home', '/{repo_name:.*?}/changelog',
                 controller='changelog', conditions=dict(function=check_repo))
 
     rmap.connect('changelog_summary_home', '/{repo_name:.*?}/changelog_summary',
                 controller='changelog', action='changelog_summary',
                 conditions=dict(function=check_repo))
 
     rmap.connect('changelog_file_home', '/{repo_name:.*?}/changelog/{revision}/{f_path:.*}',
                 controller='changelog', f_path=None,
                 conditions=dict(function=check_repo))
 
     rmap.connect('changelog_details', '/{repo_name:.*?}/changelog_details/{cs}',
                 controller='changelog', action='changelog_details',
                 conditions=dict(function=check_repo))
 
     rmap.connect('files_home', '/{repo_name:.*?}/files/{revision}/{f_path:.*}',
                 controller='files', revision='tip', f_path='',
                 conditions=dict(function=check_repo))
 
     rmap.connect('files_home_nopath', '/{repo_name:.*?}/files/{revision}',
                 controller='files', revision='tip', f_path='',
                 conditions=dict(function=check_repo))
 
     rmap.connect('files_history_home',
                  '/{repo_name:.*?}/history/{revision}/{f_path:.*}',
                  controller='files', action='history', revision='tip', f_path='',
                  conditions=dict(function=check_repo))
 
     rmap.connect('files_diff_home', '/{repo_name:.*?}/diff/{f_path:.*}',
                 controller='files', action='diff', revision='tip', f_path='',
                 conditions=dict(function=check_repo))
 
     rmap.connect('files_rawfile_home',
                  '/{repo_name:.*?}/rawfile/{revision}/{f_path:.*}',
                  controller='files', action='rawfile', revision='tip',
                  f_path='', conditions=dict(function=check_repo))
 
     rmap.connect('files_raw_home',
                  '/{repo_name:.*?}/raw/{revision}/{f_path:.*}',
                  controller='files', action='raw', revision='tip', f_path='',
                  conditions=dict(function=check_repo))
 
     rmap.connect('files_annotate_home',
                  '/{repo_name:.*?}/annotate/{revision}/{f_path:.*}',
                  controller='files', action='index', revision='tip',
                  f_path='', annotate=True, conditions=dict(function=check_repo))
 
     rmap.connect('files_edit_home',
                  '/{repo_name:.*?}/edit/{revision}/{f_path:.*}',
                  controller='files', action='edit', revision='tip',
                  f_path='', conditions=dict(function=check_repo))
 
     rmap.connect('files_add_home',
                  '/{repo_name:.*?}/add/{revision}/{f_path:.*}',
                  controller='files', action='add', revision='tip',
                  f_path='', conditions=dict(function=check_repo))
 
     rmap.connect('files_archive_home', '/{repo_name:.*?}/archive/{fname}',
                 controller='files', action='archivefile',
                 conditions=dict(function=check_repo))
 
     rmap.connect('files_nodelist_home',
                  '/{repo_name:.*?}/nodelist/{revision}/{f_path:.*}',
                 controller='files', action='nodelist',
                 conditions=dict(function=check_repo))
 
     rmap.connect('repo_fork_create_home', '/{repo_name:.*?}/fork',
                 controller='forks', action='fork_create',
                 conditions=dict(function=check_repo, method=["POST"]))
 
     rmap.connect('repo_fork_home', '/{repo_name:.*?}/fork',
                 controller='forks', action='fork',
                 conditions=dict(function=check_repo))
 
     rmap.connect('repo_forks_home', '/{repo_name:.*?}/forks',
                  controller='forks', action='forks',
                  conditions=dict(function=check_repo))
 
     rmap.connect('repo_followers_home', '/{repo_name:.*?}/followers',
                  controller='followers', action='followers',
                  conditions=dict(function=check_repo))
 
     return rmap

rhodecode/controllers/api/api.py

@@ -1,1066 +1,1109 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.api
     ~~~~~~~~~~~~~~~~~~~~~~~~~
 
     API controller for RhodeCode
 
     :created_on: Aug 20, 2011
     :author: marcink
     :copyright: (C) 2011-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
 import time
 import traceback
 import logging
 
 from rhodecode.controllers.api import JSONRPCController, JSONRPCError
 from rhodecode.lib.auth import PasswordGenerator, AuthUser, \
     HasPermissionAllDecorator, HasPermissionAnyDecorator, \
     HasPermissionAnyApi, HasRepoPermissionAnyApi
 from rhodecode.lib.utils import map_groups, repo2db_mapper
 from rhodecode.lib.utils2 import str2bool, time_to_datetime, safe_int
 from rhodecode.lib import helpers as h
 from rhodecode.model.meta import Session
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.users_group import UserGroupModel
 from rhodecode.model.db import Repository, RhodeCodeSetting, UserIpMap,\
-    Permission, User
+    Permission, User, Gist
 from rhodecode.lib.compat import json
 from rhodecode.lib.exceptions import DefaultUserException
+from rhodecode.model.gist import GistModel
 
 log = logging.getLogger(__name__)
 
 
 class OptionalAttr(object):
     """
     Special Optional Option that defines other attribute
     """
     def __init__(self, attr_name):
         self.attr_name = attr_name
 
     def __repr__(self):
         return '<OptionalAttr:%s>' % self.attr_name
 
     def __call__(self):
         return self
 #alias
 OAttr = OptionalAttr
 
 
 class Optional(object):
     """
     Defines an optional parameter::
 
         param = param.getval() if isinstance(param, Optional) else param
         param = param() if isinstance(param, Optional) else param
 
     is equivalent of::
 
         param = Optional.extract(param)
 
     """
     def __init__(self, type_):
         self.type_ = type_
 
     def __repr__(self):
         return '<Optional:%s>' % self.type_.__repr__()
 
     def __call__(self):
         return self.getval()
 
     def getval(self):
         """
         returns value from this Optional instance
         """
         return self.type_
 
     @classmethod
     def extract(cls, val):
         if isinstance(val, cls):
             return val.getval()
         return val
 
 
 def get_user_or_error(userid):
     """
     Get user by id or name or return JsonRPCError if not found
 
     :param userid:
     """
     user = UserModel().get_user(userid)
     if user is None:
         raise JSONRPCError("user `%s` does not exist" % userid)
     return user
 
 
 def get_repo_or_error(repoid):
     """
     Get repo by id or name or return JsonRPCError if not found
 
     :param userid:
     """
     repo = RepoModel().get_repo(repoid)
     if repo is None:
         raise JSONRPCError('repository `%s` does not exist' % (repoid))
     return repo
 
 
 def get_users_group_or_error(usersgroupid):
     """
     Get user group by id or name or return JsonRPCError if not found
 
     :param userid:
     """
     users_group = UserGroupModel().get_group(usersgroupid)
     if users_group is None:
         raise JSONRPCError('user group `%s` does not exist' % usersgroupid)
     return users_group
 
 
 def get_perm_or_error(permid):
     """
     Get permission by id or name or return JsonRPCError if not found
 
     :param userid:
     """
     perm = Permission.get_by_key(permid)
     if perm is None:
         raise JSONRPCError('permission `%s` does not exist' % (permid))
     return perm
 
 
 class ApiController(JSONRPCController):
     """
     API Controller
 
 
     Each method needs to have USER as argument this is then based on given
     API_KEY propagated as instance of user object
 
     Preferably this should be first argument also
 
 
     Each function should also **raise** JSONRPCError for any
     errors that happens
 
     """
 
     @HasPermissionAllDecorator('hg.admin')
     def pull(self, apiuser, repoid):
         """
         Dispatch pull action on given repo
 
         :param apiuser:
         :param repoid:
         """
 
         repo = get_repo_or_error(repoid)
 
         try:
             ScmModel().pull_changes(repo.repo_name,
                                     self.rhodecode_user.username)
             return 'Pulled from `%s`' % repo.repo_name
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'Unable to pull changes from `%s`' % repo.repo_name
             )
 
     @HasPermissionAllDecorator('hg.admin')
     def rescan_repos(self, apiuser, remove_obsolete=Optional(False)):
         """
         Dispatch rescan repositories action. If remove_obsolete is set
         than also delete repos that are in database but not in the filesystem.
         aka "clean zombies"
 
         :param apiuser:
         :param remove_obsolete:
         """
 
         try:
             rm_obsolete = Optional.extract(remove_obsolete)
             added, removed = repo2db_mapper(ScmModel().repo_scan(),
                                             remove_obsolete=rm_obsolete)
             return {'added': added, 'removed': removed}
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'Error occurred during rescan repositories action'
             )
 
     def invalidate_cache(self, apiuser, repoid):
         """
         Dispatch cache invalidation action on given repo
 
         :param apiuser:
         :param repoid:
         """
         repo = get_repo_or_error(repoid)
         if HasPermissionAnyApi('hg.admin')(user=apiuser) is False:
             # check if we have admin permission for this repo !
             if HasRepoPermissionAnyApi('repository.admin',
                                        'repository.write')(user=apiuser,
                                             repo_name=repo.repo_name) is False:
                 raise JSONRPCError('repository `%s` does not exist' % (repoid))
 
         try:
             ScmModel().mark_for_invalidation(repo.repo_name)
             return ('Caches of repository `%s` was invalidated' % repoid)
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'Error occurred during cache invalidation action'
             )
 
     def lock(self, apiuser, repoid, locked=Optional(None),
              userid=Optional(OAttr('apiuser'))):
         """
         Set locking state on particular repository by given user, if
         this command is runned by non-admin account userid is set to user
         who is calling this method
 
         :param apiuser:
         :param repoid:
         :param userid:
         :param locked:
         """
         repo = get_repo_or_error(repoid)
         if HasPermissionAnyApi('hg.admin')(user=apiuser):
             pass
         elif HasRepoPermissionAnyApi('repository.admin',
                                      'repository.write')(user=apiuser,
                                                          repo_name=repo.repo_name):
             #make sure normal user does not pass someone else userid,
             #he is not allowed to do that
             if not isinstance(userid, Optional) and userid != apiuser.user_id:
                 raise JSONRPCError(
                     'userid is not the same as your user'
                 )
         else:
             raise JSONRPCError('repository `%s` does not exist' % (repoid))
 
         if isinstance(userid, Optional):
             userid = apiuser.user_id
 
         user = get_user_or_error(userid)
 
         if isinstance(locked, Optional):
             lockobj = Repository.getlock(repo)
 
             if lockobj[0] is None:
                 _d = {
                     'repo': repo.repo_name,
                     'locked': False,
                     'locked_since': None,
                     'locked_by': None,
                     'msg': 'Repo `%s` not locked.' % repo.repo_name
                 }
                 return _d
             else:
                 userid, time_ = lockobj
                 lock_user = get_user_or_error(userid)
                 _d = {
                     'repo': repo.repo_name,
                     'locked': True,
                     'locked_since': time_,
                     'locked_by': lock_user.username,
                     'msg': ('Repo `%s` locked by `%s`. '
                             % (repo.repo_name,
                                json.dumps(time_to_datetime(time_))))
                 }
                 return _d
 
         # force locked state through a flag
         else:
             locked = str2bool(locked)
             try:
                 if locked:
                     lock_time = time.time()
                     Repository.lock(repo, user.user_id, lock_time)
                 else:
                     lock_time = None
                     Repository.unlock(repo)
                 _d = {
                     'repo': repo.repo_name,
                     'locked': locked,
                     'locked_since': lock_time,
                     'locked_by': user.username,
                     'msg': ('User `%s` set lock state for repo `%s` to `%s`'
                             % (user.username, repo.repo_name, locked))
                 }
                 return _d
             except Exception:
                 log.error(traceback.format_exc())
                 raise JSONRPCError(
                     'Error occurred locking repository `%s`' % repo.repo_name
                 )
 
     def get_locks(self, apiuser, userid=Optional(OAttr('apiuser'))):
         """
         Get all locks for given userid, if
         this command is runned by non-admin account userid is set to user
         who is calling this method, thus returning locks for himself
 
         :param apiuser:
         :param userid:
         """
         if HasPermissionAnyApi('hg.admin')(user=apiuser):
             pass
         else:
             #make sure normal user does not pass someone else userid,
             #he is not allowed to do that
             if not isinstance(userid, Optional) and userid != apiuser.user_id:
                 raise JSONRPCError(
                     'userid is not the same as your user'
                 )
         ret = []
         if isinstance(userid, Optional):
             user = None
         else:
             user = get_user_or_error(userid)
 
         #show all locks
         for r in Repository.getAll():
             userid, time_ = r.locked
             if time_:
                 _api_data = r.get_api_data()
                 # if we use userfilter just show the locks for this user
                 if user:
                     if safe_int(userid) == user.user_id:
                         ret.append(_api_data)
                 else:
                     ret.append(_api_data)
 
         return ret
 
     @HasPermissionAllDecorator('hg.admin')
     def show_ip(self, apiuser, userid):
         """
         Shows IP address as seen from RhodeCode server, together with all
         defined IP addresses for given user
 
         :param apiuser:
         :param userid:
         """
         user = get_user_or_error(userid)
         ips = UserIpMap.query().filter(UserIpMap.user == user).all()
         return dict(
             ip_addr_server=self.ip_addr,
             user_ips=ips
         )
 
     def get_user(self, apiuser, userid=Optional(OAttr('apiuser'))):
         """"
         Get a user by username, or userid, if userid is given
 
         :param apiuser:
         :param userid:
         """
         if HasPermissionAnyApi('hg.admin')(user=apiuser) is False:
             #make sure normal user does not pass someone else userid,
             #he is not allowed to do that
             if not isinstance(userid, Optional) and userid != apiuser.user_id:
                 raise JSONRPCError(
                     'userid is not the same as your user'
                 )
 
         if isinstance(userid, Optional):
             userid = apiuser.user_id
 
         user = get_user_or_error(userid)
         data = user.get_api_data()
         data['permissions'] = AuthUser(user_id=user.user_id).permissions
         return data
 
     @HasPermissionAllDecorator('hg.admin')
     def get_users(self, apiuser):
         """"
         Get all users
 
         :param apiuser:
         """
 
         result = []
         users_list = User.query().order_by(User.username)\
                         .filter(User.username != User.DEFAULT_USER)\
                         .all()
         for user in users_list:
             result.append(user.get_api_data())
         return result
 
     @HasPermissionAllDecorator('hg.admin')
     def create_user(self, apiuser, username, email, password=Optional(None),
                     firstname=Optional(None), lastname=Optional(None),
                     active=Optional(True), admin=Optional(False),
                     ldap_dn=Optional(None)):
         """
         Create new user
 
         :param apiuser:
         :param username:
         :param email:
         :param password:
         :param firstname:
         :param lastname:
         :param active:
         :param admin:
         :param ldap_dn:
         """
 
         if UserModel().get_by_username(username):
             raise JSONRPCError("user `%s` already exist" % username)
 
         if UserModel().get_by_email(email, case_insensitive=True):
             raise JSONRPCError("email `%s` already exist" % email)
 
         if Optional.extract(ldap_dn):
             # generate temporary password if ldap_dn
             password = PasswordGenerator().gen_password(length=8)
 
         try:
             user = UserModel().create_or_update(
                 username=Optional.extract(username),
                 password=Optional.extract(password),
                 email=Optional.extract(email),
                 firstname=Optional.extract(firstname),
                 lastname=Optional.extract(lastname),
                 active=Optional.extract(active),
                 admin=Optional.extract(admin),
                 ldap_dn=Optional.extract(ldap_dn)
             )
             Session().commit()
             return dict(
                 msg='created new user `%s`' % username,
                 user=user.get_api_data()
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError('failed to create user `%s`' % username)
 
     @HasPermissionAllDecorator('hg.admin')
     def update_user(self, apiuser, userid, username=Optional(None),
                     email=Optional(None), firstname=Optional(None),
                     lastname=Optional(None), active=Optional(None),
                     admin=Optional(None), ldap_dn=Optional(None),
                     password=Optional(None)):
         """
         Updates given user
 
         :param apiuser:
         :param userid:
         :param username:
         :param email:
         :param firstname:
         :param lastname:
         :param active:
         :param admin:
         :param ldap_dn:
         :param password:
         """
 
         user = get_user_or_error(userid)
 
         # call function and store only updated arguments
         updates = {}
 
         def store_update(attr, name):
             if not isinstance(attr, Optional):
                 updates[name] = attr
 
         try:
 
             store_update(username, 'username')
             store_update(password, 'password')
             store_update(email, 'email')
             store_update(firstname, 'name')
             store_update(lastname, 'lastname')
             store_update(active, 'active')
             store_update(admin, 'admin')
             store_update(ldap_dn, 'ldap_dn')
 
             user = UserModel().update_user(user, **updates)
             Session().commit()
             return dict(
                 msg='updated user ID:%s %s' % (user.user_id, user.username),
                 user=user.get_api_data()
             )
         except DefaultUserException:
             log.error(traceback.format_exc())
             raise JSONRPCError('editing default user is forbidden')
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError('failed to update user `%s`' % userid)
 
     @HasPermissionAllDecorator('hg.admin')
     def delete_user(self, apiuser, userid):
         """"
         Deletes an user
 
         :param apiuser:
         :param userid:
         """
         user = get_user_or_error(userid)
 
         try:
             UserModel().delete(userid)
             Session().commit()
             return dict(
                 msg='deleted user ID:%s %s' % (user.user_id, user.username),
                 user=None
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError('failed to delete ID:%s %s' % (user.user_id,
                                                               user.username))
 
     @HasPermissionAllDecorator('hg.admin')
     def get_users_group(self, apiuser, usersgroupid):
         """"
         Get user group by name or id
 
         :param apiuser:
         :param usersgroupid:
         """
         users_group = get_users_group_or_error(usersgroupid)
 
         data = users_group.get_api_data()
 
         members = []
         for user in users_group.members:
             user = user.user
             members.append(user.get_api_data())
         data['members'] = members
         return data
 
     @HasPermissionAllDecorator('hg.admin')
     def get_users_groups(self, apiuser):
         """"
         Get all user groups
 
         :param apiuser:
         """
 
         result = []
         for users_group in UserGroupModel().get_all():
             result.append(users_group.get_api_data())
         return result
 
     @HasPermissionAllDecorator('hg.admin')
     def create_users_group(self, apiuser, group_name,
                            owner=Optional(OAttr('apiuser')),
                            active=Optional(True)):
         """
         Creates an new usergroup
 
         :param apiuser:
         :param group_name:
         :param owner:
         :param active:
         """
 
         if UserGroupModel().get_by_name(group_name):
             raise JSONRPCError("user group `%s` already exist" % group_name)
 
         try:
             if isinstance(owner, Optional):
                 owner = apiuser.user_id
 
             owner = get_user_or_error(owner)
             active = Optional.extract(active)
             ug = UserGroupModel().create(name=group_name,
                                          owner=owner,
                                          active=active)
             Session().commit()
             return dict(
                 msg='created new user group `%s`' % group_name,
                 users_group=ug.get_api_data()
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError('failed to create group `%s`' % group_name)
 
     @HasPermissionAllDecorator('hg.admin')
     def add_user_to_users_group(self, apiuser, usersgroupid, userid):
         """"
         Add a user to a user group
 
         :param apiuser:
         :param usersgroupid:
         :param userid:
         """
         user = get_user_or_error(userid)
         users_group = get_users_group_or_error(usersgroupid)
 
         try:
             ugm = UserGroupModel().add_user_to_group(users_group, user)
             success = True if ugm != True else False
             msg = 'added member `%s` to user group `%s`' % (
                         user.username, users_group.users_group_name
                     )
             msg = msg if success else 'User is already in that group'
             Session().commit()
 
             return dict(
                 success=success,
                 msg=msg
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to add member to user group `%s`' % (
                     users_group.users_group_name
                 )
             )
 
     @HasPermissionAllDecorator('hg.admin')
     def remove_user_from_users_group(self, apiuser, usersgroupid, userid):
         """
         Remove user from a group
 
         :param apiuser:
         :param usersgroupid:
         :param userid:
         """
         user = get_user_or_error(userid)
         users_group = get_users_group_or_error(usersgroupid)
 
         try:
             success = UserGroupModel().remove_user_from_group(users_group,
                                                                user)
             msg = 'removed member `%s` from user group `%s`' % (
                         user.username, users_group.users_group_name
                     )
             msg = msg if success else "User wasn't in group"
             Session().commit()
             return dict(success=success, msg=msg)
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to remove member from user group `%s`' % (
                         users_group.users_group_name
                     )
             )
 
     def get_repo(self, apiuser, repoid):
         """"
         Get repository by name
 
         :param apiuser:
         :param repoid:
         """
         repo = get_repo_or_error(repoid)
 
         if HasPermissionAnyApi('hg.admin')(user=apiuser) is False:
             # check if we have admin permission for this repo !
             if HasRepoPermissionAnyApi('repository.admin')(user=apiuser,
                                             repo_name=repo.repo_name) is False:
                 raise JSONRPCError('repository `%s` does not exist' % (repoid))
 
         members = []
         followers = []
         for user in repo.repo_to_perm:
             perm = user.permission.permission_name
             user = user.user
             user_data = user.get_api_data()
             user_data['type'] = "user"
             user_data['permission'] = perm
             members.append(user_data)
 
         for users_group in repo.users_group_to_perm:
             perm = users_group.permission.permission_name
             users_group = users_group.users_group
             users_group_data = users_group.get_api_data()
             users_group_data['type'] = "users_group"
             users_group_data['permission'] = perm
             members.append(users_group_data)
 
         for user in repo.followers:
             followers.append(user.user.get_api_data())
 
         data = repo.get_api_data()
         data['members'] = members
         data['followers'] = followers
         return data
 
     def get_repos(self, apiuser):
         """"
         Get all repositories
 
         :param apiuser:
         """
         result = []
         if HasPermissionAnyApi('hg.admin')(user=apiuser) is False:
             repos = RepoModel().get_all_user_repos(user=apiuser)
         else:
             repos = RepoModel().get_all()
 
         for repo in repos:
             result.append(repo.get_api_data())
         return result
 
     @HasPermissionAllDecorator('hg.admin')
     def get_repo_nodes(self, apiuser, repoid, revision, root_path,
                        ret_type='all'):
         """
         returns a list of nodes and it's children
         for a given path at given revision. It's possible to specify ret_type
         to show only files or dirs
 
         :param apiuser:
         :param repoid: name or id of repository
         :param revision: revision for which listing should be done
         :param root_path: path from which start displaying
         :param ret_type: return type 'all|files|dirs' nodes
         """
         repo = get_repo_or_error(repoid)
         try:
             _d, _f = ScmModel().get_nodes(repo, revision, root_path,
                                           flat=False)
             _map = {
                 'all': _d + _f,
                 'files': _f,
                 'dirs': _d,
             }
             return _map[ret_type]
         except KeyError:
             raise JSONRPCError('ret_type must be one of %s' % _map.keys())
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to get repo: `%s` nodes' % repo.repo_name
             )
 
     @HasPermissionAnyDecorator('hg.admin', 'hg.create.repository')
     def create_repo(self, apiuser, repo_name, owner=Optional(OAttr('apiuser')),
                     repo_type=Optional('hg'),
                     description=Optional(''), private=Optional(False),
                     clone_uri=Optional(None), landing_rev=Optional('tip'),
                     enable_statistics=Optional(False),
                     enable_locking=Optional(False),
                     enable_downloads=Optional(False)):
         """
         Create repository, if clone_url is given it makes a remote clone
         if repo_name is within a group name the groups will be created
         automatically if they aren't present
 
         :param apiuser:
         :param repo_name:
         :param onwer:
         :param repo_type:
         :param description:
         :param private:
         :param clone_uri:
         :param landing_rev:
         """
         if HasPermissionAnyApi('hg.admin')(user=apiuser) is False:
             if not isinstance(owner, Optional):
                 #forbid setting owner for non-admins
                 raise JSONRPCError(
                     'Only RhodeCode admin can specify `owner` param'
                 )
         if isinstance(owner, Optional):
             owner = apiuser.user_id
 
         owner = get_user_or_error(owner)
 
         if RepoModel().get_by_repo_name(repo_name):
             raise JSONRPCError("repo `%s` already exist" % repo_name)
 
         defs = RhodeCodeSetting.get_default_repo_settings(strip_prefix=True)
         if isinstance(private, Optional):
             private = defs.get('repo_private') or Optional.extract(private)
         if isinstance(repo_type, Optional):
             repo_type = defs.get('repo_type')
         if isinstance(enable_statistics, Optional):
             enable_statistics = defs.get('repo_enable_statistics')
         if isinstance(enable_locking, Optional):
             enable_locking = defs.get('repo_enable_locking')
         if isinstance(enable_downloads, Optional):
             enable_downloads = defs.get('repo_enable_downloads')
 
         clone_uri = Optional.extract(clone_uri)
         description = Optional.extract(description)
         landing_rev = Optional.extract(landing_rev)
 
         try:
             # create structure of groups and return the last group
             group = map_groups(repo_name)
 
             repo = RepoModel().create_repo(
                 repo_name=repo_name,
                 repo_type=repo_type,
                 description=description,
                 owner=owner,
                 private=private,
                 clone_uri=clone_uri,
                 repos_group=group,
                 landing_rev=landing_rev,
                 enable_statistics=enable_statistics,
                 enable_downloads=enable_downloads,
                 enable_locking=enable_locking
             )
 
             Session().commit()
             return dict(
                 msg="Created new repository `%s`" % (repo.repo_name),
                 repo=repo.get_api_data()
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError('failed to create repository `%s`' % repo_name)
 
     @HasPermissionAnyDecorator('hg.admin', 'hg.fork.repository')
     def fork_repo(self, apiuser, repoid, fork_name, owner=Optional(OAttr('apiuser')),
                   description=Optional(''), copy_permissions=Optional(False),
                   private=Optional(False), landing_rev=Optional('tip')):
         repo = get_repo_or_error(repoid)
         repo_name = repo.repo_name
 
         _repo = RepoModel().get_by_repo_name(fork_name)
         if _repo:
             type_ = 'fork' if _repo.fork else 'repo'
             raise JSONRPCError("%s `%s` already exist" % (type_, fork_name))
 
         if HasPermissionAnyApi('hg.admin')(user=apiuser):
             pass
         elif HasRepoPermissionAnyApi('repository.admin',
                                      'repository.write',
                                      'repository.read')(user=apiuser,
                                                         repo_name=repo.repo_name):
             if not isinstance(owner, Optional):
                 #forbid setting owner for non-admins
                 raise JSONRPCError(
                     'Only RhodeCode admin can specify `owner` param'
                 )
         else:
             raise JSONRPCError('repository `%s` does not exist' % (repoid))
 
         if isinstance(owner, Optional):
             owner = apiuser.user_id
 
         owner = get_user_or_error(owner)
 
         try:
             # create structure of groups and return the last group
             group = map_groups(fork_name)
 
             form_data = dict(
                 repo_name=fork_name,
                 repo_name_full=fork_name,
                 repo_group=group,
                 repo_type=repo.repo_type,
                 description=Optional.extract(description),
                 private=Optional.extract(private),
                 copy_permissions=Optional.extract(copy_permissions),
                 landing_rev=Optional.extract(landing_rev),
                 update_after_clone=False,
                 fork_parent_id=repo.repo_id,
             )
             RepoModel().create_fork(form_data, cur_user=owner)
             return dict(
                 msg='Created fork of `%s` as `%s`' % (repo.repo_name,
                                                       fork_name),
                 success=True  # cannot return the repo data here since fork
                               # cann be done async
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to fork repository `%s` as `%s`' % (repo_name,
                                                             fork_name)
             )
 
+    # perms handled inside
     def delete_repo(self, apiuser, repoid, forks=Optional(None)):
         """
         Deletes a given repository
 
         :param apiuser:
         :param repoid:
         :param forks: detach or delete, what do do with attached forks for repo
         """
         repo = get_repo_or_error(repoid)
 
         if HasPermissionAnyApi('hg.admin')(user=apiuser) is False:
             # check if we have admin permission for this repo !
             if HasRepoPermissionAnyApi('repository.admin')(user=apiuser,
                                             repo_name=repo.repo_name) is False:
                 raise JSONRPCError('repository `%s` does not exist' % (repoid))
 
         try:
             handle_forks = Optional.extract(forks)
             _forks_msg = ''
             _forks = [f for f in repo.forks]
             if handle_forks == 'detach':
                 _forks_msg = ' ' + _('Detached %s forks') % len(_forks)
             elif handle_forks == 'delete':
                 _forks_msg = ' ' + _('Deleted %s forks') % len(_forks)
             elif _forks:
                 raise JSONRPCError(
                     'Cannot delete `%s` it still contains attached forks'
                     % repo.repo_name
                 )
 
             RepoModel().delete(repo, forks=forks)
             Session().commit()
             return dict(
                 msg='Deleted repository `%s`%s' % (repo.repo_name, _forks_msg),
                 success=True
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to delete repository `%s`' % repo.repo_name
             )
 
     @HasPermissionAllDecorator('hg.admin')
     def grant_user_permission(self, apiuser, repoid, userid, perm):
         """
         Grant permission for user on given repository, or update existing one
         if found
 
         :param repoid:
         :param userid:
         :param perm:
         """
         repo = get_repo_or_error(repoid)
         user = get_user_or_error(userid)
         perm = get_perm_or_error(perm)
 
         try:
 
             RepoModel().grant_user_permission(repo=repo, user=user, perm=perm)
 
             Session().commit()
             return dict(
                 msg='Granted perm: `%s` for user: `%s` in repo: `%s`' % (
                     perm.permission_name, user.username, repo.repo_name
                 ),
                 success=True
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to edit permission for user: `%s` in repo: `%s`' % (
                     userid, repoid
                 )
             )
 
     @HasPermissionAllDecorator('hg.admin')
     def revoke_user_permission(self, apiuser, repoid, userid):
         """
         Revoke permission for user on given repository
 
         :param apiuser:
         :param repoid:
         :param userid:
         """
 
         repo = get_repo_or_error(repoid)
         user = get_user_or_error(userid)
         try:
 
             RepoModel().revoke_user_permission(repo=repo, user=user)
 
             Session().commit()
             return dict(
                 msg='Revoked perm for user: `%s` in repo: `%s`' % (
                     user.username, repo.repo_name
                 ),
                 success=True
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to edit permission for user: `%s` in repo: `%s`' % (
                     userid, repoid
                 )
             )
 
     @HasPermissionAllDecorator('hg.admin')
     def grant_users_group_permission(self, apiuser, repoid, usersgroupid,
                                      perm):
         """
         Grant permission for user group on given repository, or update
         existing one if found
 
         :param apiuser:
         :param repoid:
         :param usersgroupid:
         :param perm:
         """
         repo = get_repo_or_error(repoid)
         perm = get_perm_or_error(perm)
         users_group = get_users_group_or_error(usersgroupid)
 
         try:
             RepoModel().grant_users_group_permission(repo=repo,
                                                      group_name=users_group,
                                                      perm=perm)
 
             Session().commit()
             return dict(
                 msg='Granted perm: `%s` for user group: `%s` in '
                     'repo: `%s`' % (
                     perm.permission_name, users_group.users_group_name,
                     repo.repo_name
                 ),
                 success=True
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to edit permission for user group: `%s` in '
                 'repo: `%s`' % (
                     usersgroupid, repo.repo_name
                 )
             )
 
     @HasPermissionAllDecorator('hg.admin')
     def revoke_users_group_permission(self, apiuser, repoid, usersgroupid):
         """
         Revoke permission for user group on given repository
 
         :param apiuser:
         :param repoid:
         :param usersgroupid:
         """
         repo = get_repo_or_error(repoid)
         users_group = get_users_group_or_error(usersgroupid)
 
         try:
             RepoModel().revoke_users_group_permission(repo=repo,
                                                       group_name=users_group)
 
             Session().commit()
             return dict(
                 msg='Revoked perm for user group: `%s` in repo: `%s`' % (
                     users_group.users_group_name, repo.repo_name
                 ),
                 success=True
             )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to edit permission for user group: `%s` in '
                 'repo: `%s`' % (
                     users_group.users_group_name, repo.repo_name
                 )
             )
+
+    def create_gist(self, apiuser, files, owner=Optional(OAttr('apiuser')),
+                    gist_type=Optional(Gist.GIST_PUBLIC),
+                    gist_lifetime=Optional(-1),
+                    gist_description=Optional('')):
+
+        try:
+            if isinstance(owner, Optional):
+                owner = apiuser.user_id
+
+            owner = get_user_or_error(owner)
+            description = Optional.extract(gist_description)
+            gist_type = Optional.extract(gist_type)
+            gist_lifetime = Optional.extract(gist_lifetime)
+
+            # files: {
+            #    'filename': {'content':'...', 'lexer': null},
+            #    'filename2': {'content':'...', 'lexer': null}
+            #}
+            gist = GistModel().create(description=description,
+                                      owner=owner,
+                                      gist_mapping=files,
+                                      gist_type=gist_type,
+                                      lifetime=gist_lifetime)
+            Session().commit()
+            return dict(
+                msg='created new gist',
+                gist_url=gist.gist_url(),
+                gist_id=gist.gist_access_id,
+                gist_type=gist.gist_type,
+                files=files.keys()
+            )
+        except Exception:
+            log.error(traceback.format_exc())
+            raise JSONRPCError('failed to create gist')
+
+    def update_gist(self, apiuser):
+        pass
+
+    def delete_gist(self, apiuser):
+        pass

rhodecode/controllers/files.py

@@ -1,650 +1,658 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.files
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Files controller for RhodeCode
 
     :created_on: Apr 21, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import with_statement
 import os
 import logging
 import traceback
 import tempfile
 import shutil
 
 from pylons import request, response, tmpl_context as c, url
 from pylons.i18n.translation import _
 from pylons.controllers.util import redirect
 from rhodecode.lib.utils import jsonify
 
 from rhodecode.lib import diffs
 from rhodecode.lib import helpers as h
 
 from rhodecode.lib.compat import OrderedDict
 from rhodecode.lib.utils2 import convert_line_endings, detect_mode, safe_str,\
     str2bool
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.vcs.backends.base import EmptyChangeset
 from rhodecode.lib.vcs.conf import settings
 from rhodecode.lib.vcs.exceptions import RepositoryError, \
     ChangesetDoesNotExistError, EmptyRepositoryError, \
     ImproperArchiveTypeError, VCSError, NodeAlreadyExistsError,\
     NodeDoesNotExistError, ChangesetError, NodeError
 from rhodecode.lib.vcs.nodes import FileNode
 
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.db import Repository
 
 from rhodecode.controllers.changeset import anchor_url, _ignorews_url,\
     _context_url, get_line_ctx, get_ignore_ws
 from webob.exc import HTTPNotFound
+from rhodecode.lib.exceptions import NonRelativePathError
 
 
 log = logging.getLogger(__name__)
 
 
 class FilesController(BaseRepoController):
 
     def __before__(self):
         super(FilesController, self).__before__()
         c.cut_off_limit = self.cut_off_limit
 
     def __get_cs_or_redirect(self, rev, repo_name, redirect_after=True):
         """
         Safe way to get changeset if error occur it redirects to tip with
         proper message
 
         :param rev: revision to fetch
         :param repo_name: repo name to redirect after
         """
 
         try:
             return c.rhodecode_repo.get_changeset(rev)
         except EmptyRepositoryError, e:
             if not redirect_after:
                 return None
             url_ = url('files_add_home',
                        repo_name=c.repo_name,
                        revision=0, f_path='')
             add_new = h.link_to(_('Click here to add new file'), url_)
             h.flash(h.literal(_('There are no files yet %s') % add_new),
                     category='warning')
             redirect(h.url('summary_home', repo_name=repo_name))
 
         except RepositoryError, e:  # including ChangesetDoesNotExistError
             h.flash(str(e), category='error')
             raise HTTPNotFound()
 
     def __get_filenode_or_redirect(self, repo_name, cs, path):
         """
         Returns file_node, if error occurs or given path is directory,
         it'll redirect to top level path
 
         :param repo_name: repo_name
         :param cs: given changeset
         :param path: path to lookup
         """
 
         try:
             file_node = cs.get_node(path)
             if file_node.is_dir():
                 raise RepositoryError('given path is a directory')
         except RepositoryError, e:
             h.flash(str(e), category='error')
             raise HTTPNotFound()
 
         return file_node
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def index(self, repo_name, revision, f_path, annotate=False):
         # redirect to given revision from form if given
         post_revision = request.POST.get('at_rev', None)
         if post_revision:
             cs = self.__get_cs_or_redirect(post_revision, repo_name)
 
         c.changeset = self.__get_cs_or_redirect(revision, repo_name)
         c.branch = request.GET.get('branch', None)
         c.f_path = f_path
         c.annotate = annotate
         c.changeset = self.__get_cs_or_redirect(revision, repo_name)
         cur_rev = c.changeset.revision
 
         # prev link
         try:
             prev_rev = c.rhodecode_repo.get_changeset(cur_rev).prev(c.branch)
             c.url_prev = url('files_home', repo_name=c.repo_name,
                          revision=prev_rev.raw_id, f_path=f_path)
             if c.branch:
                 c.url_prev += '?branch=%s' % c.branch
         except (ChangesetDoesNotExistError, VCSError):
             c.url_prev = '#'
 
         # next link
         try:
             next_rev = c.rhodecode_repo.get_changeset(cur_rev).next(c.branch)
             c.url_next = url('files_home', repo_name=c.repo_name,
                      revision=next_rev.raw_id, f_path=f_path)
             if c.branch:
                 c.url_next += '?branch=%s' % c.branch
         except (ChangesetDoesNotExistError, VCSError):
             c.url_next = '#'
 
         # files or dirs
         try:
             c.file = c.changeset.get_node(f_path)
 
             if c.file.is_file():
                 c.load_full_history = False
                 file_last_cs = c.file.last_changeset
                 c.file_changeset = (c.changeset
                                     if c.changeset.revision < file_last_cs.revision
                                     else file_last_cs)
                 #determine if we're on branch head
                 _branches = c.rhodecode_repo.branches
                 c.on_branch_head = revision in _branches.keys() + _branches.values()
                 _hist = []
                 c.file_history = []
                 if c.load_full_history:
                     c.file_history, _hist = self._get_node_history(c.changeset, f_path)
 
                 c.authors = []
                 for a in set([x.author for x in _hist]):
                     c.authors.append((h.email(a), h.person(a)))
             else:
                 c.authors = c.file_history = []
         except RepositoryError, e:
             h.flash(str(e), category='error')
             raise HTTPNotFound()
 
         if request.environ.get('HTTP_X_PARTIAL_XHR'):
             return render('files/files_ypjax.html')
 
         return render('files/files.html')
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def history(self, repo_name, revision, f_path, annotate=False):
         if request.environ.get('HTTP_X_PARTIAL_XHR'):
             c.changeset = self.__get_cs_or_redirect(revision, repo_name)
             c.f_path = f_path
             c.annotate = annotate
             c.file = c.changeset.get_node(f_path)
             if c.file.is_file():
                 file_last_cs = c.file.last_changeset
                 c.file_changeset = (c.changeset
                                     if c.changeset.revision < file_last_cs.revision
                                     else file_last_cs)
                 c.file_history, _hist = self._get_node_history(c.changeset, f_path)
                 c.authors = []
                 for a in set([x.author for x in _hist]):
                     c.authors.append((h.email(a), h.person(a)))
                 return render('files/files_history_box.html')
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def rawfile(self, repo_name, revision, f_path):
         cs = self.__get_cs_or_redirect(revision, repo_name)
         file_node = self.__get_filenode_or_redirect(repo_name, cs, f_path)
 
         response.content_disposition = 'attachment; filename=%s' % \
             safe_str(f_path.split(Repository.url_sep())[-1])
 
         response.content_type = file_node.mimetype
         return file_node.content
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def raw(self, repo_name, revision, f_path):
         cs = self.__get_cs_or_redirect(revision, repo_name)
         file_node = self.__get_filenode_or_redirect(repo_name, cs, f_path)
 
         raw_mimetype_mapping = {
             # map original mimetype to a mimetype used for "show as raw"
             # you can also provide a content-disposition to override the
             # default "attachment" disposition.
             # orig_type: (new_type, new_dispo)
 
             # show images inline:
             'image/x-icon': ('image/x-icon', 'inline'),
             'image/png': ('image/png', 'inline'),
             'image/gif': ('image/gif', 'inline'),
             'image/jpeg': ('image/jpeg', 'inline'),
             'image/svg+xml': ('image/svg+xml', 'inline'),
         }
 
         mimetype = file_node.mimetype
         try:
             mimetype, dispo = raw_mimetype_mapping[mimetype]
         except KeyError:
             # we don't know anything special about this, handle it safely
             if file_node.is_binary:
                 # do same as download raw for binary files
                 mimetype, dispo = 'application/octet-stream', 'attachment'
             else:
                 # do not just use the original mimetype, but force text/plain,
                 # otherwise it would serve text/html and that might be unsafe.
                 # Note: underlying vcs library fakes text/plain mimetype if the
                 # mimetype can not be determined and it thinks it is not
                 # binary.This might lead to erroneous text display in some
                 # cases, but helps in other cases, like with text files
                 # without extension.
                 mimetype, dispo = 'text/plain', 'inline'
 
         if dispo == 'attachment':
             dispo = 'attachment; filename=%s' % \
                         safe_str(f_path.split(os.sep)[-1])
 
         response.content_disposition = dispo
         response.content_type = mimetype
         return file_node.content
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     def edit(self, repo_name, revision, f_path):
         repo = c.rhodecode_db_repo
         if repo.enable_locking and repo.locked[0]:
             h.flash(_('This repository is has been locked by %s on %s')
                 % (h.person_by_id(repo.locked[0]),
                    h.fmt_date(h.time_to_datetime(repo.locked[1]))),
                   'warning')
             return redirect(h.url('files_home',
                                   repo_name=repo_name, revision='tip'))
 
         # check if revision is a branch identifier- basically we cannot
         # create multiple heads via file editing
         _branches = repo.scm_instance.branches
         # check if revision is a branch name or branch hash
         if revision not in _branches.keys() + _branches.values():
             h.flash(_('You can only edit files with revision '
                       'being a valid branch '), category='warning')
             return redirect(h.url('files_home',
                                   repo_name=repo_name, revision='tip',
                                   f_path=f_path))
 
         r_post = request.POST
 
         c.cs = self.__get_cs_or_redirect(revision, repo_name)
         c.file = self.__get_filenode_or_redirect(repo_name, c.cs, f_path)
 
         if c.file.is_binary:
             return redirect(url('files_home', repo_name=c.repo_name,
                          revision=c.cs.raw_id, f_path=f_path))
         c.default_message = _('Edited file %s via RhodeCode') % (f_path)
         c.f_path = f_path
 
         if r_post:
 
             old_content = c.file.content
             sl = old_content.splitlines(1)
             first_line = sl[0] if sl else ''
             # modes:  0 - Unix, 1 - Mac, 2 - DOS
             mode = detect_mode(first_line, 0)
             content = convert_line_endings(r_post.get('content', ''), mode)
 
             message = r_post.get('message') or c.default_message
             author = self.rhodecode_user.full_contact
 
             if content == old_content:
                 h.flash(_('No changes'), category='warning')
                 return redirect(url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
             try:
                 self.scm_model.commit_change(repo=c.rhodecode_repo,
                                              repo_name=repo_name, cs=c.cs,
                                              user=self.rhodecode_user.user_id,
                                              author=author, message=message,
                                              content=content, f_path=f_path)
                 h.flash(_('Successfully committed to %s') % f_path,
                         category='success')
 
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred during commit'), category='error')
             return redirect(url('changeset_home',
                                 repo_name=c.repo_name, revision='tip'))
 
         return render('files/files_edit.html')
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     def add(self, repo_name, revision, f_path):
 
         repo = Repository.get_by_repo_name(repo_name)
         if repo.enable_locking and repo.locked[0]:
             h.flash(_('This repository is has been locked by %s on %s')
                 % (h.person_by_id(repo.locked[0]),
                    h.fmt_date(h.time_to_datetime(repo.locked[1]))),
                   'warning')
             return redirect(h.url('files_home',
                                   repo_name=repo_name, revision='tip'))
 
         r_post = request.POST
         c.cs = self.__get_cs_or_redirect(revision, repo_name,
                                          redirect_after=False)
         if c.cs is None:
             c.cs = EmptyChangeset(alias=c.rhodecode_repo.alias)
         c.default_message = (_('Added file via RhodeCode'))
         c.f_path = f_path
 
         if r_post:
             unix_mode = 0
             content = convert_line_endings(r_post.get('content', ''), unix_mode)
 
             message = r_post.get('message') or c.default_message
             filename = r_post.get('filename')
             location = r_post.get('location', '')
             file_obj = r_post.get('upload_file', None)
 
             if file_obj is not None and hasattr(file_obj, 'filename'):
                 filename = file_obj.filename
                 content = file_obj.file
 
             if not content:
                 h.flash(_('No content'), category='warning')
                 return redirect(url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
             if not filename:
                 h.flash(_('No filename'), category='warning')
                 return redirect(url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
-            if location.startswith('/') or location.startswith('.') or '../' in location:
-                h.flash(_('Location must be relative path and must not '
-                          'contain .. in path'), category='warning')
-                return redirect(url('changeset_home', repo_name=c.repo_name,
-                                    revision='tip'))
-            if location:
-                location = os.path.normpath(location)
+            #strip all crap out of file, just leave the basename
             filename = os.path.basename(filename)
             node_path = os.path.join(location, filename)
             author = self.rhodecode_user.full_contact
 
             try:
-                self.scm_model.create_node(repo=c.rhodecode_repo,
-                                           repo_name=repo_name, cs=c.cs,
-                                           user=self.rhodecode_user.user_id,
-                                           author=author, message=message,
-                                           content=content, f_path=node_path)
+                nodes = {
+                    node_path: {
+                        'content': content
+                    }
+                }
+                self.scm_model.create_nodes(
+                    user=c.rhodecode_user.user_id, repo=c.rhodecode_db_repo,
+                    message=message,
+                    nodes=nodes,
+                    parent_cs=c.cs,
+                    author=author,
+                )
+
                 h.flash(_('Successfully committed to %s') % node_path,
                         category='success')
+            except NonRelativePathError, e:
+                h.flash(_('Location must be relative path and must not '
+                          'contain .. in path'), category='warning')
+                return redirect(url('changeset_home', repo_name=c.repo_name,
+                                    revision='tip'))
             except (NodeError, NodeAlreadyExistsError), e:
                 h.flash(_(e), category='error')
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred during commit'), category='error')
             return redirect(url('changeset_home',
                                 repo_name=c.repo_name, revision='tip'))
 
         return render('files/files_add.html')
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def archivefile(self, repo_name, fname):
 
         fileformat = None
         revision = None
         ext = None
         subrepos = request.GET.get('subrepos') == 'true'
 
         for a_type, ext_data in settings.ARCHIVE_SPECS.items():
             archive_spec = fname.split(ext_data[1])
             if len(archive_spec) == 2 and archive_spec[1] == '':
                 fileformat = a_type or ext_data[1]
                 revision = archive_spec[0]
                 ext = ext_data[1]
 
         try:
             dbrepo = RepoModel().get_by_repo_name(repo_name)
             if not dbrepo.enable_downloads:
                 return _('Downloads disabled')
 
             if c.rhodecode_repo.alias == 'hg':
                 # patch and reset hooks section of UI config to not run any
                 # hooks on fetching archives with subrepos
                 for k, v in c.rhodecode_repo._repo.ui.configitems('hooks'):
                     c.rhodecode_repo._repo.ui.setconfig('hooks', k, None)
 
             cs = c.rhodecode_repo.get_changeset(revision)
             content_type = settings.ARCHIVE_SPECS[fileformat][0]
         except ChangesetDoesNotExistError:
             return _('Unknown revision %s') % revision
         except EmptyRepositoryError:
             return _('Empty repository')
         except (ImproperArchiveTypeError, KeyError):
             return _('Unknown archive type')
         # archive cache
         from rhodecode import CONFIG
         rev_name = cs.raw_id[:12]
         archive_name = '%s-%s%s' % (safe_str(repo_name.replace('/', '_')),
                                     safe_str(rev_name), ext)
 
         use_cached_archive = False  # defines if we use cached version of archive
         archive_cache_enabled = CONFIG.get('archive_cache_dir')
         if not subrepos and archive_cache_enabled:
             #check if we it's ok to write
             if not os.path.isdir(CONFIG['archive_cache_dir']):
                 os.makedirs(CONFIG['archive_cache_dir'])
             cached_archive_path = os.path.join(CONFIG['archive_cache_dir'], archive_name)
             if os.path.isfile(cached_archive_path):
                 log.debug('Found cached archive in %s' % cached_archive_path)
                 fd, archive = None, cached_archive_path
                 use_cached_archive = True
             else:
                 log.debug('Archive %s is not yet cached' % (archive_name))
 
         if not use_cached_archive:
             #generate new archive
             try:
                 fd, archive = tempfile.mkstemp()
                 t = open(archive, 'wb')
                 log.debug('Creating new temp archive in %s' % archive)
                 cs.fill_archive(stream=t, kind=fileformat, subrepos=subrepos)
                 if archive_cache_enabled:
                     #if we generated the archive and use cache rename that
                     log.debug('Storing new archive in %s' % cached_archive_path)
                     shutil.move(archive, cached_archive_path)
                     archive = cached_archive_path
             finally:
                 t.close()
 
         def get_chunked_archive(archive):
             stream = open(archive, 'rb')
             while True:
                 data = stream.read(16 * 1024)
                 if not data:
                     stream.close()
                     if fd:  # fd means we used temporary file
                         os.close(fd)
                     if not archive_cache_enabled:
                         log.debug('Destroing temp archive %s' % archive)
                         os.remove(archive)
                     break
                 yield data
 
         response.content_disposition = str('attachment; filename=%s' % (archive_name))
         response.content_type = str(content_type)
         return get_chunked_archive(archive)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def diff(self, repo_name, f_path):
         ignore_whitespace = request.GET.get('ignorews') == '1'
         line_context = request.GET.get('context', 3)
         diff1 = request.GET.get('diff1', '')
         diff2 = request.GET.get('diff2', '')
         c.action = request.GET.get('diff')
         c.no_changes = diff1 == diff2
         c.f_path = f_path
         c.big_diff = False
         c.anchor_url = anchor_url
         c.ignorews_url = _ignorews_url
         c.context_url = _context_url
         c.changes = OrderedDict()
         c.changes[diff2] = []
 
         #special case if we want a show rev only, it's impl here
         #to reduce JS and callbacks
 
         if request.GET.get('show_rev'):
             if str2bool(request.GET.get('annotate', 'False')):
                 _url = url('files_annotate_home', repo_name=c.repo_name,
                            revision=diff1, f_path=c.f_path)
             else:
                 _url = url('files_home', repo_name=c.repo_name,
                            revision=diff1, f_path=c.f_path)
 
             return redirect(_url)
         try:
             if diff1 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_1 = c.rhodecode_repo.get_changeset(diff1)
                 try:
                     node1 = c.changeset_1.get_node(f_path)
                     if node1.is_dir():
                         raise NodeError('%s path is a %s not a file'
                                         % (node1, type(node1)))
                 except NodeDoesNotExistError:
                     c.changeset_1 = EmptyChangeset(cs=diff1,
                                                    revision=c.changeset_1.revision,
                                                    repo=c.rhodecode_repo)
                     node1 = FileNode(f_path, '', changeset=c.changeset_1)
             else:
                 c.changeset_1 = EmptyChangeset(repo=c.rhodecode_repo)
                 node1 = FileNode(f_path, '', changeset=c.changeset_1)
 
             if diff2 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_2 = c.rhodecode_repo.get_changeset(diff2)
                 try:
                     node2 = c.changeset_2.get_node(f_path)
                     if node2.is_dir():
                         raise NodeError('%s path is a %s not a file'
                                         % (node2, type(node2)))
                 except NodeDoesNotExistError:
                     c.changeset_2 = EmptyChangeset(cs=diff2,
                                                    revision=c.changeset_2.revision,
                                                    repo=c.rhodecode_repo)
                     node2 = FileNode(f_path, '', changeset=c.changeset_2)
             else:
                 c.changeset_2 = EmptyChangeset(repo=c.rhodecode_repo)
                 node2 = FileNode(f_path, '', changeset=c.changeset_2)
         except (RepositoryError, NodeError):
             log.error(traceback.format_exc())
             return redirect(url('files_home', repo_name=c.repo_name,
                                 f_path=f_path))
 
         if c.action == 'download':
             _diff = diffs.get_gitdiff(node1, node2,
                                       ignore_whitespace=ignore_whitespace,
                                       context=line_context)
             diff = diffs.DiffProcessor(_diff, format='gitdiff')
 
             diff_name = '%s_vs_%s.diff' % (diff1, diff2)
             response.content_type = 'text/plain'
             response.content_disposition = (
                 'attachment; filename=%s' % diff_name
             )
             return diff.as_raw()
 
         elif c.action == 'raw':
             _diff = diffs.get_gitdiff(node1, node2,
                                       ignore_whitespace=ignore_whitespace,
                                       context=line_context)
             diff = diffs.DiffProcessor(_diff, format='gitdiff')
             response.content_type = 'text/plain'
             return diff.as_raw()
 
         else:
             fid = h.FID(diff2, node2.path)
             line_context_lcl = get_line_ctx(fid, request.GET)
             ign_whitespace_lcl = get_ignore_ws(fid, request.GET)
 
             lim = request.GET.get('fulldiff') or self.cut_off_limit
             _, cs1, cs2, diff, st = diffs.wrapped_diff(filenode_old=node1,
                                          filenode_new=node2,
                                          cut_off_limit=lim,
                                          ignore_whitespace=ign_whitespace_lcl,
                                          line_context=line_context_lcl,
                                          enable_comments=False)
             op = ''
             filename = node1.path
             cs_changes = {
                 'fid': [cs1, cs2, op, filename, diff, st]
             }
             c.changes = cs_changes
 
         return render('files/file_diff.html')
 
     def _get_node_history(self, cs, f_path, changesets=None):
         """
         get changesets history for given node
 
         :param cs: changeset to calculate history
         :param f_path: path for node to calculate history for
         :param changesets: if passed don't calculate history and take
             changesets defined in this list
         """
         # calculate history based on tip
         tip_cs = c.rhodecode_repo.get_changeset()
         if changesets is None:
             try:
                 changesets = tip_cs.get_file_history(f_path)
             except (NodeDoesNotExistError, ChangesetError):
                 #this node is not present at tip !
                 changesets = cs.get_file_history(f_path)
         hist_l = []
 
         changesets_group = ([], _("Changesets"))
         branches_group = ([], _("Branches"))
         tags_group = ([], _("Tags"))
         _hg = cs.repository.alias == 'hg'
         for chs in changesets:
             #_branch = '(%s)' % chs.branch if _hg else ''
             _branch = chs.branch
             n_desc = 'r%s:%s (%s)' % (chs.revision, chs.short_id, _branch)
             changesets_group[0].append((chs.raw_id, n_desc,))
         hist_l.append(changesets_group)
 
         for name, chs in c.rhodecode_repo.branches.items():
             branches_group[0].append((chs, name),)
         hist_l.append(branches_group)
 
         for name, chs in c.rhodecode_repo.tags.items():
             tags_group[0].append((chs, name),)
         hist_l.append(tags_group)
 
         return hist_l, changesets
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     @jsonify
     def nodelist(self, repo_name, revision, f_path):
         if request.environ.get('HTTP_X_PARTIAL_XHR'):
             cs = self.__get_cs_or_redirect(revision, repo_name)
             _d, _f = ScmModel().get_nodes(repo_name, cs.raw_id, f_path,
                                           flat=False)
             return {'nodes': _d + _f}

rhodecode/controllers/pullrequests.py

@@ -1,560 +1,559 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.pullrequests
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     pull requests controller for rhodecode for initializing pull requests
 
     :created_on: May 7, 2012
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import logging
 import traceback
 import formencode
 
 from webob.exc import HTTPNotFound, HTTPForbidden
 from collections import defaultdict
 from itertools import groupby
 
 from pylons import request, response, session, tmpl_context as c, url
 from pylons.controllers.util import abort, redirect
 from pylons.i18n.translation import _
 
 from rhodecode.lib.compat import json
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator,\
     NotAnonymous
 from rhodecode.lib.helpers import Page
 from rhodecode.lib import helpers as h
 from rhodecode.lib import diffs
 from rhodecode.lib.utils import action_logger, jsonify
 from rhodecode.lib.vcs.utils import safe_str
 from rhodecode.lib.vcs.exceptions import EmptyRepositoryError
 from rhodecode.lib.vcs.backends.base import EmptyChangeset
 from rhodecode.lib.diffs import LimitedDiffContainer
 from rhodecode.model.db import User, PullRequest, ChangesetStatus,\
     ChangesetComment
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.meta import Session
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.comment import ChangesetCommentsModel
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.forms import PullRequestForm
 from mercurial import scmutil
 from rhodecode.lib.utils2 import safe_int
 
 log = logging.getLogger(__name__)
 
 
 class PullrequestsController(BaseRepoController):
 
     def __before__(self):
         super(PullrequestsController, self).__before__()
         repo_model = RepoModel()
         c.users_array = repo_model.get_users_js()
         c.users_groups_array = repo_model.get_users_groups_js()
 
     def _get_repo_refs(self, repo, rev=None, branch=None, branch_rev=None):
         """return a structure with repo's interesting changesets, suitable for
         the selectors in pullrequest.html
 
         rev: a revision that must be in the list somehow and selected by default
         branch: a branch that must be in the list and selected by default - even if closed
         branch_rev: a revision of which peers should be preferred and available."""
         # list named branches that has been merged to this named branch - it should probably merge back
         peers = []
 
         if rev:
             rev = safe_str(rev)
 
         if branch:
             branch = safe_str(branch)
 
         if branch_rev:
             branch_rev = safe_str(branch_rev)
             # not restricting to merge() would also get branch point and be better
             # (especially because it would get the branch point) ... but is currently too expensive
             otherbranches = {}
             for i in repo._repo.revs(
                 "sort(parents(branch(id(%s)) and merge()) - branch(id(%s)))",
                 branch_rev, branch_rev):
                 cs = repo.get_changeset(i)
                 otherbranches[cs.branch] = cs.raw_id
             for abranch, node in otherbranches.iteritems():
                 selected = 'branch:%s:%s' % (abranch, node)
                 peers.append((selected, abranch))
 
         selected = None
 
         branches = []
         for abranch, branchrev in repo.branches.iteritems():
             n = 'branch:%s:%s' % (abranch, branchrev)
             branches.append((n, abranch))
             if rev == branchrev:
                 selected = n
             if branch == abranch:
                 selected = n
                 branch = None
         if branch: # branch not in list - it is probably closed
             revs = repo._repo.revs('max(branch(%s))', branch)
             if revs:
                 cs = repo.get_changeset(revs[0])
                 selected = 'branch:%s:%s' % (branch, cs.raw_id)
                 branches.append((selected, branch))
 
         bookmarks = []
         for bookmark, bookmarkrev in repo.bookmarks.iteritems():
             n = 'book:%s:%s' % (bookmark, bookmarkrev)
             bookmarks.append((n, bookmark))
             if rev == bookmarkrev:
                 selected = n
 
         tags = []
         for tag, tagrev in repo.tags.iteritems():
             n = 'tag:%s:%s' % (tag, tagrev)
             tags.append((n, tag))
             if rev == tagrev and tag != 'tip': # tip is not a real tag - and its branch is better
                 selected = n
 
         # prio 1: rev was selected as existing entry above
 
         # prio 2: create special entry for rev; rev _must_ be used
         specials = []
         if rev and selected is None:
             selected = 'rev:%s:%s' % (rev, rev)
             specials = [(selected, '%s: %s' % (_("Changeset"), rev[:12]))]
 
         # prio 3: most recent peer branch
         if peers and not selected:
             selected = peers[0][0][0]
 
         # prio 4: tip revision
         if not selected:
             selected = 'tag:tip:%s' % repo.tags['tip']
 
         groups = [(specials, _("Special")),
                   (peers, _("Peer branches")),
                   (bookmarks, _("Bookmarks")),
                   (branches, _("Branches")),
                   (tags, _("Tags")),
                   ]
         return [g for g in groups if g[0]], selected
 
     def _get_is_allowed_change_status(self, pull_request):
         owner = self.rhodecode_user.user_id == pull_request.user_id
         reviewer = self.rhodecode_user.user_id in [x.user_id for x in
                                                    pull_request.reviewers]
         return (self.rhodecode_user.admin or owner or reviewer)
 
     def _load_compare_data(self, pull_request, enable_comments=True):
         """
         Load context data needed for generating compare diff
 
         :param pull_request:
-        :type pull_request:
         """
         org_repo = pull_request.org_repo
         (org_ref_type,
          org_ref_name,
          org_ref_rev) = pull_request.org_ref.split(':')
 
         other_repo = org_repo
         (other_ref_type,
          other_ref_name,
          other_ref_rev) = pull_request.other_ref.split(':')
 
         # despite opening revisions for bookmarks/branches/tags, we always
         # convert this to rev to prevent changes after bookmark or branch change
         org_ref = ('rev', org_ref_rev)
         other_ref = ('rev', other_ref_rev)
 
         c.org_repo = org_repo
         c.other_repo = other_repo
 
         c.fulldiff = fulldiff = request.GET.get('fulldiff')
 
         c.cs_ranges = [org_repo.get_changeset(x) for x in pull_request.revisions]
 
         c.statuses = org_repo.statuses([x.raw_id for x in c.cs_ranges])
 
         c.org_ref = org_ref[1]
         c.org_ref_type = org_ref[0]
         c.other_ref = other_ref[1]
         c.other_ref_type = other_ref[0]
 
         diff_limit = self.cut_off_limit if not fulldiff else None
 
         # we swap org/other ref since we run a simple diff on one repo
         log.debug('running diff between %s and %s in %s'
                   % (other_ref, org_ref, org_repo.scm_instance.path))
         txtdiff = org_repo.scm_instance.get_diff(rev1=safe_str(other_ref[1]), rev2=safe_str(org_ref[1]))
 
         diff_processor = diffs.DiffProcessor(txtdiff or '', format='gitdiff',
                                              diff_limit=diff_limit)
         _parsed = diff_processor.prepare()
 
         c.limited_diff = False
         if isinstance(_parsed, LimitedDiffContainer):
             c.limited_diff = True
 
         c.files = []
         c.changes = {}
         c.lines_added = 0
         c.lines_deleted = 0
 
         for f in _parsed:
             st = f['stats']
             c.lines_added += st['added']
             c.lines_deleted += st['deleted']
             fid = h.FID('', f['filename'])
             c.files.append([fid, f['operation'], f['filename'], f['stats']])
             htmldiff = diff_processor.as_html(enable_comments=enable_comments,
                                               parsed_lines=[f])
             c.changes[fid] = [f['operation'], f['filename'], htmldiff]
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def show_all(self, repo_name):
         c.pull_requests = PullRequestModel().get_all(repo_name)
         c.repo_name = repo_name
         p = safe_int(request.GET.get('page', 1), 1)
 
         c.pullrequests_pager = Page(c.pull_requests, page=p, items_per_page=10)
 
         c.pullrequest_data = render('/pullrequests/pullrequest_data.html')
 
         if request.environ.get('HTTP_X_PARTIAL_XHR'):
             return c.pullrequest_data
 
         return render('/pullrequests/pullrequest_show_all.html')
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def index(self):
         org_repo = c.rhodecode_db_repo
 
         if org_repo.scm_instance.alias != 'hg':
             log.error('Review not available for GIT REPOS')
             raise HTTPNotFound
 
         try:
             org_repo.scm_instance.get_changeset()
         except EmptyRepositoryError, e:
             h.flash(h.literal(_('There are no changesets yet')),
                     category='warning')
             redirect(url('summary_home', repo_name=org_repo.repo_name))
 
         org_rev = request.GET.get('rev_end')
         # rev_start is not directly useful - its parent could however be used
         # as default for other and thus give a simple compare view
         #other_rev = request.POST.get('rev_start')
         branch = request.GET.get('branch')
 
         c.org_repos = []
         c.org_repos.append((org_repo.repo_name, org_repo.repo_name))
         c.default_org_repo = org_repo.repo_name
         c.org_refs, c.default_org_ref = self._get_repo_refs(org_repo.scm_instance, rev=org_rev, branch=branch)
 
         c.other_repos = []
         other_repos_info = {}
 
         def add_other_repo(repo, branch_rev=None):
             if repo.repo_name in other_repos_info: # shouldn't happen
                 return
             c.other_repos.append((repo.repo_name, repo.repo_name))
             other_refs, selected_other_ref = self._get_repo_refs(repo.scm_instance, branch_rev=branch_rev)
             other_repos_info[repo.repo_name] = {
                 'user': dict(user_id=repo.user.user_id,
                              username=repo.user.username,
                              firstname=repo.user.firstname,
                              lastname=repo.user.lastname,
                              gravatar_link=h.gravatar_url(repo.user.email, 14)),
                 'description': repo.description.split('\n', 1)[0],
                 'revs': h.select('other_ref', selected_other_ref, other_refs, class_='refs')
             }
 
         # add org repo to other so we can open pull request against peer branches on itself
         add_other_repo(org_repo, branch_rev=org_rev)
         c.default_other_repo = org_repo.repo_name
 
         # gather forks and add to this list ... even though it is rare to
         # request forks to pull from their parent
         for fork in org_repo.forks:
             add_other_repo(fork)
 
         # add parents of this fork also, but only if it's not empty
         if org_repo.parent and org_repo.parent.scm_instance.revisions:
             add_other_repo(org_repo.parent)
             c.default_other_repo = org_repo.parent.repo_name
 
         c.default_other_repo_info = other_repos_info[c.default_other_repo]
         c.other_repos_info = json.dumps(other_repos_info)
 
         return render('/pullrequests/pullrequest.html')
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def create(self, repo_name):
         repo = RepoModel()._get_repo(repo_name)
         try:
             _form = PullRequestForm(repo.repo_id)().to_python(request.POST)
         except formencode.Invalid, errors:
             log.error(traceback.format_exc())
             if errors.error_dict.get('revisions'):
                 msg = 'Revisions: %s' % errors.error_dict['revisions']
             elif errors.error_dict.get('pullrequest_title'):
                 msg = _('Pull request requires a title with min. 3 chars')
             else:
                 msg = _('Error creating pull request')
 
             h.flash(msg, 'error')
             return redirect(url('pullrequest_home', repo_name=repo_name))
 
         org_repo = _form['org_repo']
         org_ref = 'rev:merge:%s' % _form['merge_rev']
         other_repo = _form['other_repo']
         other_ref = 'rev:ancestor:%s' % _form['ancestor_rev']
         revisions = [x for x in reversed(_form['revisions'])]
         reviewers = _form['review_members']
 
         title = _form['pullrequest_title']
         description = _form['pullrequest_desc']
         try:
             pull_request = PullRequestModel().create(
                 self.rhodecode_user.user_id, org_repo, org_ref, other_repo,
                 other_ref, revisions, reviewers, title, description
             )
             Session().commit()
             h.flash(_('Successfully opened new pull request'),
                     category='success')
         except Exception:
             h.flash(_('Error occurred during sending pull request'),
                     category='error')
             log.error(traceback.format_exc())
             return redirect(url('pullrequest_home', repo_name=repo_name))
 
         return redirect(url('pullrequest_show', repo_name=other_repo,
                             pull_request_id=pull_request.pull_request_id))
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     @jsonify
     def update(self, repo_name, pull_request_id):
         pull_request = PullRequest.get_or_404(pull_request_id)
         if pull_request.is_closed():
             raise HTTPForbidden()
         #only owner or admin can update it
         owner = pull_request.author.user_id == c.rhodecode_user.user_id
         if h.HasPermissionAny('hg.admin', 'repository.admin')() or owner:
             reviewers_ids = map(int, filter(lambda v: v not in [None, ''],
                        request.POST.get('reviewers_ids', '').split(',')))
 
             PullRequestModel().update_reviewers(pull_request_id, reviewers_ids)
             Session().commit()
             return True
         raise HTTPForbidden()
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     @jsonify
     def delete(self, repo_name, pull_request_id):
         pull_request = PullRequest.get_or_404(pull_request_id)
         #only owner can delete it !
         if pull_request.author.user_id == c.rhodecode_user.user_id:
             PullRequestModel().delete(pull_request)
             Session().commit()
             h.flash(_('Successfully deleted pull request'),
                     category='success')
             return redirect(url('admin_settings_my_account', anchor='pullrequests'))
         raise HTTPForbidden()
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def show(self, repo_name, pull_request_id):
         repo_model = RepoModel()
         c.users_array = repo_model.get_users_js()
         c.users_groups_array = repo_model.get_users_groups_js()
         c.pull_request = PullRequest.get_or_404(pull_request_id)
         c.allowed_to_change_status = self._get_is_allowed_change_status(c.pull_request)
         cc_model = ChangesetCommentsModel()
         cs_model = ChangesetStatusModel()
         _cs_statuses = cs_model.get_statuses(c.pull_request.org_repo,
                                             pull_request=c.pull_request,
                                             with_revisions=True)
 
         cs_statuses = defaultdict(list)
         for st in _cs_statuses:
             cs_statuses[st.author.username] += [st]
 
         c.pull_request_reviewers = []
         c.pull_request_pending_reviewers = []
         for o in c.pull_request.reviewers:
             st = cs_statuses.get(o.user.username, None)
             if st:
                 sorter = lambda k: k.version
                 st = [(x, list(y)[0])
                       for x, y in (groupby(sorted(st, key=sorter), sorter))]
             else:
                 c.pull_request_pending_reviewers.append(o.user)
             c.pull_request_reviewers.append([o.user, st])
 
         # pull_requests repo_name we opened it against
         # ie. other_repo must match
         if repo_name != c.pull_request.other_repo.repo_name:
             raise HTTPNotFound
 
         # load compare data into template context
         enable_comments = not c.pull_request.is_closed()
         self._load_compare_data(c.pull_request, enable_comments=enable_comments)
 
         # inline comments
         c.inline_cnt = 0
         c.inline_comments = cc_model.get_inline_comments(
                                 c.rhodecode_db_repo.repo_id,
                                 pull_request=pull_request_id)
         # count inline comments
         for __, lines in c.inline_comments:
             for comments in lines.values():
                 c.inline_cnt += len(comments)
         # comments
         c.comments = cc_model.get_comments(c.rhodecode_db_repo.repo_id,
                                            pull_request=pull_request_id)
 
         try:
             cur_status = c.statuses[c.pull_request.revisions[0]][0]
         except Exception:
             log.error(traceback.format_exc())
             cur_status = 'undefined'
         if c.pull_request.is_closed() and 0:
             c.current_changeset_status = cur_status
         else:
             # changeset(pull-request) status calulation based on reviewers
             c.current_changeset_status = cs_model.calculate_status(
                                             c.pull_request_reviewers,
                                          )
         c.changeset_statuses = ChangesetStatus.STATUSES
 
         c.as_form = False
         c.ancestor = None # there is one - but right here we don't know which
         return render('/pullrequests/pullrequest_show.html')
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     @jsonify
     def comment(self, repo_name, pull_request_id):
         pull_request = PullRequest.get_or_404(pull_request_id)
         if pull_request.is_closed():
             raise HTTPForbidden()
 
         status = request.POST.get('changeset_status')
         change_status = request.POST.get('change_changeset_status')
         text = request.POST.get('text')
         close_pr = request.POST.get('save_close')
 
         allowed_to_change_status = self._get_is_allowed_change_status(pull_request)
         if status and change_status and allowed_to_change_status:
             _def = (_('Status change -> %s')
                             % ChangesetStatus.get_status_lbl(status))
             if close_pr:
                 _def = _('Closing with') + ' ' + _def
             text = text or _def
         comm = ChangesetCommentsModel().create(
             text=text,
             repo=c.rhodecode_db_repo.repo_id,
             user=c.rhodecode_user.user_id,
             pull_request=pull_request_id,
             f_path=request.POST.get('f_path'),
             line_no=request.POST.get('line'),
             status_change=(ChangesetStatus.get_status_lbl(status)
                            if status and change_status
                            and allowed_to_change_status else None),
             closing_pr=close_pr
         )
 
         action_logger(self.rhodecode_user,
                       'user_commented_pull_request:%s' % pull_request_id,
                       c.rhodecode_db_repo, self.ip_addr, self.sa)
 
         if allowed_to_change_status:
             # get status if set !
             if status and change_status:
                 ChangesetStatusModel().set_status(
                     c.rhodecode_db_repo.repo_id,
                     status,
                     c.rhodecode_user.user_id,
                     comm,
                     pull_request=pull_request_id
                 )
 
             if close_pr:
                 if status in ['rejected', 'approved']:
                     PullRequestModel().close_pull_request(pull_request_id)
                     action_logger(self.rhodecode_user,
                               'user_closed_pull_request:%s' % pull_request_id,
                               c.rhodecode_db_repo, self.ip_addr, self.sa)
                 else:
                     h.flash(_('Closing pull request on other statuses than '
                               'rejected or approved forbidden'),
                             category='warning')
 
         Session().commit()
 
         if not request.environ.get('HTTP_X_PARTIAL_XHR'):
             return redirect(h.url('pullrequest_show', repo_name=repo_name,
                                   pull_request_id=pull_request_id))
 
         data = {
            'target_id': h.safeid(h.safe_unicode(request.POST.get('f_path'))),
         }
         if comm:
             c.co = comm
             data.update(comm.get_dict())
             data.update({'rendered_text':
                          render('changeset/changeset_comment_block.html')})
 
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     @jsonify
     def delete_comment(self, repo_name, comment_id):
         co = ChangesetComment.get(comment_id)
         if co.pull_request.is_closed():
             #don't allow deleting comments on closed pull request
             raise HTTPForbidden()
 
         owner = co.author.user_id == c.rhodecode_user.user_id
         if h.HasPermissionAny('hg.admin', 'repository.admin')() or owner:
             ChangesetCommentsModel().delete(comment=co)
             Session().commit()
             return True
         else:
             raise HTTPForbidden()

rhodecode/lib/db_manage.py

@@ -1,726 +1,725 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.db_manage
     ~~~~~~~~~~~~~~~~~~~~~~~
 
     Database creation, and setup module for RhodeCode. Used for creation
     of database as well as for migration operations
 
     :created_on: Apr 10, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import sys
 import uuid
 import logging
 from os.path import dirname as dn, join as jn
 
 from rhodecode import __dbversion__, __py_version__
 
 from rhodecode.model.user import UserModel
 from rhodecode.lib.utils import ask_ok
 from rhodecode.model import init_model
 from rhodecode.model.db import User, Permission, RhodeCodeUi, \
     RhodeCodeSetting, UserToPerm, DbMigrateVersion, RepoGroup, \
     UserRepoGroupToPerm, CacheInvalidation, UserGroup
 
 from sqlalchemy.engine import create_engine
 from rhodecode.model.repos_group import ReposGroupModel
 #from rhodecode.model import meta
 from rhodecode.model.meta import Session, Base
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.users_group import UserGroupModel
 
 
 log = logging.getLogger(__name__)
 
 
 def notify(msg):
     """
     Notification for migrations messages
     """
     ml = len(msg) + (4 * 2)
     print >> sys.stdout, ('*** %s ***\n%s' % (msg, '*' * ml)).upper()
 
 
 class DbManage(object):
     def __init__(self, log_sql, dbconf, root, tests=False, cli_args={}):
         self.dbname = dbconf.split('/')[-1]
         self.tests = tests
         self.root = root
         self.dburi = dbconf
         self.log_sql = log_sql
         self.db_exists = False
         self.cli_args = cli_args
         self.init_db()
 
         force_ask = self.cli_args.get('force_ask')
         if force_ask is not None:
             global ask_ok
             ask_ok = lambda *args, **kwargs: force_ask
 
     def init_db(self):
         engine = create_engine(self.dburi, echo=self.log_sql)
         init_model(engine)
         self.sa = Session()
 
     def create_tables(self, override=False):
         """
         Create a auth database
         """
 
         log.info("Any existing database is going to be destroyed")
         if self.tests:
             destroy = True
         else:
             destroy = ask_ok('Are you sure to destroy old database ? [y/n]')
         if not destroy:
             sys.exit('Nothing tables created')
         if destroy:
             Base.metadata.drop_all()
 
         checkfirst = not override
         Base.metadata.create_all(checkfirst=checkfirst)
         log.info('Created tables for %s' % self.dbname)
 
     def set_db_version(self):
         ver = DbMigrateVersion()
         ver.version = __dbversion__
         ver.repository_id = 'rhodecode_db_migrations'
         ver.repository_path = 'versions'
         self.sa.add(ver)
         log.info('db version set to: %s' % __dbversion__)
 
     def upgrade(self):
         """
         Upgrades given database schema to given revision following
         all needed steps, to perform the upgrade
 
         """
 
         from rhodecode.lib.dbmigrate.migrate.versioning import api
         from rhodecode.lib.dbmigrate.migrate.exceptions import \
             DatabaseNotControlledError
 
         if 'sqlite' in self.dburi:
             print (
                '********************** WARNING **********************\n'
                'Make sure your version of sqlite is at least 3.7.X.  \n'
                'Earlier versions are known to fail on some migrations\n'
                '*****************************************************\n'
             )
         upgrade = ask_ok('You are about to perform database upgrade, make '
                          'sure You backed up your database before. '
                          'Continue ? [y/n]')
         if not upgrade:
             sys.exit('No upgrade performed')
 
         repository_path = jn(dn(dn(dn(os.path.realpath(__file__)))),
                              'rhodecode/lib/dbmigrate')
         db_uri = self.dburi
 
         try:
             curr_version = api.db_version(db_uri, repository_path)
             msg = ('Found current database under version'
                  ' control with version %s' % curr_version)
 
         except (RuntimeError, DatabaseNotControlledError):
             curr_version = 1
             msg = ('Current database is not under version control. Setting'
                    ' as version %s' % curr_version)
             api.version_control(db_uri, repository_path, curr_version)
 
         notify(msg)
 
         if curr_version == __dbversion__:
             sys.exit('This database is already at the newest version')
 
         # clear cache keys
         log.info("Clearing cache keys now...")
         CacheInvalidation.clear_cache()
 
         #======================================================================
         # UPGRADE STEPS
         #======================================================================
 
         class UpgradeSteps(object):
             """
             Those steps follow schema versions so for example schema
             for example schema with seq 002 == step_2 and so on.
             """
 
             def __init__(self, klass):
                 self.klass = klass
 
             def step_0(self):
                 # step 0 is the schema upgrade, and than follow proper upgrades
                 notify('attempting to do database upgrade from '
                        'version %s to version %s' %(curr_version, __dbversion__))
                 api.upgrade(db_uri, repository_path, __dbversion__)
                 notify('Schema upgrade completed')
 
             def step_1(self):
                 pass
 
             def step_2(self):
                 notify('Patching repo paths for newer version of RhodeCode')
                 self.klass.fix_repo_paths()
 
                 notify('Patching default user of RhodeCode')
                 self.klass.fix_default_user()
 
                 log.info('Changing ui settings')
                 self.klass.create_ui_settings()
 
             def step_3(self):
                 notify('Adding additional settings into RhodeCode db')
                 self.klass.fix_settings()
                 notify('Adding ldap defaults')
                 self.klass.create_ldap_options(skip_existing=True)
 
             def step_4(self):
                 notify('create permissions and fix groups')
                 self.klass.create_permissions()
                 self.klass.fixup_groups()
 
             def step_5(self):
                 pass
 
             def step_6(self):
 
                 notify('re-checking permissions')
                 self.klass.create_permissions()
 
                 notify('installing new UI options')
                 sett4 = RhodeCodeSetting('show_public_icon', True)
                 Session().add(sett4)
                 sett5 = RhodeCodeSetting('show_private_icon', True)
                 Session().add(sett5)
                 sett6 = RhodeCodeSetting('stylify_metatags', False)
                 Session().add(sett6)
 
                 notify('fixing old PULL hook')
                 _pull = RhodeCodeUi.get_by_key('preoutgoing.pull_logger')
                 if _pull:
                     _pull.ui_key = RhodeCodeUi.HOOK_PULL
                     Session().add(_pull)
 
                 notify('fixing old PUSH hook')
                 _push = RhodeCodeUi.get_by_key('pretxnchangegroup.push_logger')
                 if _push:
                     _push.ui_key = RhodeCodeUi.HOOK_PUSH
                     Session().add(_push)
 
                 notify('installing new pre-push hook')
                 hooks4 = RhodeCodeUi()
                 hooks4.ui_section = 'hooks'
                 hooks4.ui_key = RhodeCodeUi.HOOK_PRE_PUSH
                 hooks4.ui_value = 'python:rhodecode.lib.hooks.pre_push'
                 Session().add(hooks4)
 
                 notify('installing new pre-pull hook')
                 hooks6 = RhodeCodeUi()
                 hooks6.ui_section = 'hooks'
                 hooks6.ui_key = RhodeCodeUi.HOOK_PRE_PULL
                 hooks6.ui_value = 'python:rhodecode.lib.hooks.pre_pull'
                 Session().add(hooks6)
 
                 notify('installing hgsubversion option')
                 # enable hgsubversion disabled by default
                 hgsubversion = RhodeCodeUi()
                 hgsubversion.ui_section = 'extensions'
                 hgsubversion.ui_key = 'hgsubversion'
                 hgsubversion.ui_value = ''
                 hgsubversion.ui_active = False
                 Session().add(hgsubversion)
 
                 notify('installing hg git option')
                 # enable hggit disabled by default
                 hggit = RhodeCodeUi()
                 hggit.ui_section = 'extensions'
                 hggit.ui_key = 'hggit'
                 hggit.ui_value = ''
                 hggit.ui_active = False
                 Session().add(hggit)
 
                 notify('re-check default permissions')
                 default_user = User.get_by_username(User.DEFAULT_USER)
                 perm = Permission.get_by_key('hg.fork.repository')
                 reg_perm = UserToPerm()
                 reg_perm.user = default_user
                 reg_perm.permission = perm
                 Session().add(reg_perm)
 
             def step_7(self):
                 perm_fixes = self.klass.reset_permissions(User.DEFAULT_USER)
                 Session().commit()
                 if perm_fixes:
                     notify('There was an inconsistent state of permissions '
                            'detected for default user. Permissions are now '
                            'reset to the default value for default user. '
                            'Please validate and check default permissions '
                            'in admin panel')
 
             def step_8(self):
                 self.klass.create_permissions()
                 self.klass.populate_default_permissions()
                 self.klass.create_default_options(skip_existing=True)
                 Session().commit()
 
             def step_9(self):
                 perm_fixes = self.klass.reset_permissions(User.DEFAULT_USER)
                 Session().commit()
                 if perm_fixes:
                     notify('There was an inconsistent state of permissions '
                            'detected for default user. Permissions are now '
                            'reset to the default value for default user. '
                            'Please validate and check default permissions '
                            'in admin panel')
 
             def step_10(self):
                 pass
 
             def step_11(self):
                 self.klass.update_repo_info()
 
             def step_12(self):
                 self.klass.create_permissions()
                 Session().commit()
 
                 self.klass.populate_default_permissions()
                 Session().commit()
 
                 #fix all usergroups
                 ug_model = UserGroupModel()
                 for ug in UserGroup.get_all():
                     perm_obj = ug_model._create_default_perms(ug)
                     Session().add(perm_obj)
                 Session().commit()
 
         upgrade_steps = [0] + range(curr_version + 1, __dbversion__ + 1)
 
         # CALL THE PROPER ORDER OF STEPS TO PERFORM FULL UPGRADE
         _step = None
         for step in upgrade_steps:
             notify('performing upgrade step %s' % step)
             getattr(UpgradeSteps(self), 'step_%s' % step)()
             self.sa.commit()
             _step = step
 
         notify('upgrade to version %s successful' % _step)
 
     def fix_repo_paths(self):
         """
         Fixes a old rhodecode version path into new one without a '*'
         """
 
         paths = self.sa.query(RhodeCodeUi)\
                 .filter(RhodeCodeUi.ui_key == '/')\
                 .scalar()
 
         paths.ui_value = paths.ui_value.replace('*', '')
 
         try:
             self.sa.add(paths)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def fix_default_user(self):
         """
         Fixes a old default user with some 'nicer' default values,
         used mostly for anonymous access
         """
         def_user = self.sa.query(User)\
                 .filter(User.username == 'default')\
                 .one()
 
         def_user.name = 'Anonymous'
         def_user.lastname = 'User'
         def_user.email = '[e-mail]'
 
         try:
             self.sa.add(def_user)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def fix_settings(self):
         """
         Fixes rhodecode settings adds ga_code key for google analytics
         """
 
         hgsettings3 = RhodeCodeSetting('ga_code', '')
 
         try:
             self.sa.add(hgsettings3)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def admin_prompt(self, second=False):
         if not self.tests:
             import getpass
 
             # defaults
             defaults = self.cli_args
             username = defaults.get('username')
             password = defaults.get('password')
             email = defaults.get('email')
 
             def get_password():
                 password = getpass.getpass('Specify admin password '
                                            '(min 6 chars):')
                 confirm = getpass.getpass('Confirm password:')
 
                 if password != confirm:
                     log.error('passwords mismatch')
                     return False
                 if len(password) < 6:
                     log.error('password is to short use at least 6 characters')
                     return False
 
                 return password
             if username is None:
                 username = raw_input('Specify admin username:')
             if password is None:
                 password = get_password()
                 if not password:
                     #second try
                     password = get_password()
                     if not password:
                         sys.exit()
             if email is None:
                 email = raw_input('Specify admin email:')
             self.create_user(username, password, email, True)
         else:
             log.info('creating admin and regular test users')
             from rhodecode.tests import TEST_USER_ADMIN_LOGIN, \
             TEST_USER_ADMIN_PASS, TEST_USER_ADMIN_EMAIL, \
             TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS, \
             TEST_USER_REGULAR_EMAIL, TEST_USER_REGULAR2_LOGIN, \
             TEST_USER_REGULAR2_PASS, TEST_USER_REGULAR2_EMAIL
 
             self.create_user(TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS,
                              TEST_USER_ADMIN_EMAIL, True)
 
             self.create_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS,
                              TEST_USER_REGULAR_EMAIL, False)
 
             self.create_user(TEST_USER_REGULAR2_LOGIN, TEST_USER_REGULAR2_PASS,
                              TEST_USER_REGULAR2_EMAIL, False)
 
     def create_ui_settings(self):
         """
         Creates ui settings, fills out hooks
         and disables dotencode
         """
 
         #HOOKS
         hooks1_key = RhodeCodeUi.HOOK_UPDATE
         hooks1_ = self.sa.query(RhodeCodeUi)\
             .filter(RhodeCodeUi.ui_key == hooks1_key).scalar()
 
         hooks1 = RhodeCodeUi() if hooks1_ is None else hooks1_
         hooks1.ui_section = 'hooks'
         hooks1.ui_key = hooks1_key
         hooks1.ui_value = 'hg update >&2'
         hooks1.ui_active = False
         self.sa.add(hooks1)
 
         hooks2_key = RhodeCodeUi.HOOK_REPO_SIZE
         hooks2_ = self.sa.query(RhodeCodeUi)\
             .filter(RhodeCodeUi.ui_key == hooks2_key).scalar()
         hooks2 = RhodeCodeUi() if hooks2_ is None else hooks2_
         hooks2.ui_section = 'hooks'
         hooks2.ui_key = hooks2_key
         hooks2.ui_value = 'python:rhodecode.lib.hooks.repo_size'
         self.sa.add(hooks2)
 
         hooks3 = RhodeCodeUi()
         hooks3.ui_section = 'hooks'
         hooks3.ui_key = RhodeCodeUi.HOOK_PUSH
         hooks3.ui_value = 'python:rhodecode.lib.hooks.log_push_action'
         self.sa.add(hooks3)
 
         hooks4 = RhodeCodeUi()
         hooks4.ui_section = 'hooks'
         hooks4.ui_key = RhodeCodeUi.HOOK_PRE_PUSH
         hooks4.ui_value = 'python:rhodecode.lib.hooks.pre_push'
         self.sa.add(hooks4)
 
         hooks5 = RhodeCodeUi()
         hooks5.ui_section = 'hooks'
         hooks5.ui_key = RhodeCodeUi.HOOK_PULL
         hooks5.ui_value = 'python:rhodecode.lib.hooks.log_pull_action'
         self.sa.add(hooks5)
 
         hooks6 = RhodeCodeUi()
         hooks6.ui_section = 'hooks'
         hooks6.ui_key = RhodeCodeUi.HOOK_PRE_PULL
         hooks6.ui_value = 'python:rhodecode.lib.hooks.pre_pull'
         self.sa.add(hooks6)
 
         # enable largefiles
         largefiles = RhodeCodeUi()
         largefiles.ui_section = 'extensions'
         largefiles.ui_key = 'largefiles'
         largefiles.ui_value = ''
         self.sa.add(largefiles)
 
         # enable hgsubversion disabled by default
         hgsubversion = RhodeCodeUi()
         hgsubversion.ui_section = 'extensions'
         hgsubversion.ui_key = 'hgsubversion'
         hgsubversion.ui_value = ''
         hgsubversion.ui_active = False
         self.sa.add(hgsubversion)
 
         # enable hggit disabled by default
         hggit = RhodeCodeUi()
         hggit.ui_section = 'extensions'
         hggit.ui_key = 'hggit'
         hggit.ui_value = ''
         hggit.ui_active = False
         self.sa.add(hggit)
 
     def create_ldap_options(self, skip_existing=False):
         """Creates ldap settings"""
 
         for k, v in [('ldap_active', 'false'), ('ldap_host', ''),
                     ('ldap_port', '389'), ('ldap_tls_kind', 'PLAIN'),
                     ('ldap_tls_reqcert', ''), ('ldap_dn_user', ''),
                     ('ldap_dn_pass', ''), ('ldap_base_dn', ''),
                     ('ldap_filter', ''), ('ldap_search_scope', ''),
                     ('ldap_attr_login', ''), ('ldap_attr_firstname', ''),
                     ('ldap_attr_lastname', ''), ('ldap_attr_email', '')]:
 
             if skip_existing and RhodeCodeSetting.get_by_name(k) != None:
                 log.debug('Skipping option %s' % k)
                 continue
             setting = RhodeCodeSetting(k, v)
             self.sa.add(setting)
 
     def create_default_options(self, skip_existing=False):
         """Creates default settings"""
 
         for k, v in [
             ('default_repo_enable_locking',  False),
             ('default_repo_enable_downloads', False),
             ('default_repo_enable_statistics', False),
             ('default_repo_private', False),
             ('default_repo_type', 'hg')]:
 
             if skip_existing and RhodeCodeSetting.get_by_name(k) != None:
                 log.debug('Skipping option %s' % k)
                 continue
             setting = RhodeCodeSetting(k, v)
             self.sa.add(setting)
 
     def fixup_groups(self):
         def_usr = User.get_default_user()
         for g in RepoGroup.query().all():
             g.group_name = g.get_new_name(g.name)
             self.sa.add(g)
             # get default perm
             default = UserRepoGroupToPerm.query()\
                 .filter(UserRepoGroupToPerm.group == g)\
                 .filter(UserRepoGroupToPerm.user == def_usr)\
                 .scalar()
 
             if default is None:
                 log.debug('missing default permission for group %s adding' % g)
                 perm_obj = ReposGroupModel()._create_default_perms(g)
                 self.sa.add(perm_obj)
 
     def reset_permissions(self, username):
         """
         Resets permissions to default state, usefull when old systems had
         bad permissions, we must clean them up
 
         :param username:
-        :type username:
         """
         default_user = User.get_by_username(username)
         if not default_user:
             return
 
         u2p = UserToPerm.query()\
             .filter(UserToPerm.user == default_user).all()
         fixed = False
         if len(u2p) != len(Permission.DEFAULT_USER_PERMISSIONS):
             for p in u2p:
                 Session().delete(p)
             fixed = True
             self.populate_default_permissions()
         return fixed
 
     def update_repo_info(self):
         RepoModel.update_repoinfo()
 
     def config_prompt(self, test_repo_path='', retries=3):
         defaults = self.cli_args
         _path = defaults.get('repos_location')
         if retries == 3:
             log.info('Setting up repositories config')
 
         if _path is not None:
             path = _path
         elif not self.tests and not test_repo_path:
             path = raw_input(
                  'Enter a valid absolute path to store repositories. '
                  'All repositories in that path will be added automatically:'
             )
         else:
             path = test_repo_path
         path_ok = True
 
         # check proper dir
         if not os.path.isdir(path):
             path_ok = False
             log.error('Given path %s is not a valid directory' % path)
 
         elif not os.path.isabs(path):
             path_ok = False
             log.error('Given path %s is not an absolute path' % path)
 
         # check write access
         elif not os.access(path, os.W_OK) and path_ok:
             path_ok = False
             log.error('No write permission to given path %s' % path)
 
         if retries == 0:
             sys.exit('max retries reached')
         if not path_ok:
             retries -= 1
             return self.config_prompt(test_repo_path, retries)
 
         real_path = os.path.normpath(os.path.realpath(path))
 
         if real_path != os.path.normpath(path):
             if not ask_ok(('Path looks like a symlink, Rhodecode will store '
                            'given path as %s ? [y/n]') % (real_path)):
                 log.error('Canceled by user')
                 sys.exit(-1)
 
         return real_path
 
     def create_settings(self, path):
 
         self.create_ui_settings()
 
         #HG UI OPTIONS
         web1 = RhodeCodeUi()
         web1.ui_section = 'web'
         web1.ui_key = 'push_ssl'
         web1.ui_value = 'false'
 
         web2 = RhodeCodeUi()
         web2.ui_section = 'web'
         web2.ui_key = 'allow_archive'
         web2.ui_value = 'gz zip bz2'
 
         web3 = RhodeCodeUi()
         web3.ui_section = 'web'
         web3.ui_key = 'allow_push'
         web3.ui_value = '*'
 
         web4 = RhodeCodeUi()
         web4.ui_section = 'web'
         web4.ui_key = 'baseurl'
         web4.ui_value = '/'
 
         paths = RhodeCodeUi()
         paths.ui_section = 'paths'
         paths.ui_key = '/'
         paths.ui_value = path
 
         phases = RhodeCodeUi()
         phases.ui_section = 'phases'
         phases.ui_key = 'publish'
         phases.ui_value = False
 
         sett1 = RhodeCodeSetting('realm', 'RhodeCode authentication')
         sett2 = RhodeCodeSetting('title', 'RhodeCode')
         sett3 = RhodeCodeSetting('ga_code', '')
 
         sett4 = RhodeCodeSetting('show_public_icon', True)
         sett5 = RhodeCodeSetting('show_private_icon', True)
         sett6 = RhodeCodeSetting('stylify_metatags', False)
 
         self.sa.add(web1)
         self.sa.add(web2)
         self.sa.add(web3)
         self.sa.add(web4)
         self.sa.add(paths)
         self.sa.add(sett1)
         self.sa.add(sett2)
         self.sa.add(sett3)
         self.sa.add(sett4)
         self.sa.add(sett5)
         self.sa.add(sett6)
 
         self.create_ldap_options()
         self.create_default_options()
 
         log.info('created ui config')
 
     def create_user(self, username, password, email='', admin=False):
         log.info('creating user %s' % username)
         UserModel().create_or_update(username, password, email,
                                      firstname='RhodeCode', lastname='Admin',
                                      active=True, admin=admin)
 
     def create_default_user(self):
         log.info('creating default user')
         # create default user for handling default permissions.
         UserModel().create_or_update(username='default',
                               password=str(uuid.uuid1())[:8],
                               email='[e-mail]',
                               firstname='Anonymous', lastname='User')
 
     def create_permissions(self):
         """
         Creates all permissions defined in the system
         """
         # module.(access|create|change|delete)_[name]
         # module.(none|read|write|admin)
         log.info('creating permissions')
         PermissionModel(self.sa).create_permissions()
 
     def populate_default_permissions(self):
         """
         Populate default permissions. It will create only the default
         permissions that are missing, and not alter already defined ones
         """
         log.info('creating default user permissions')
         PermissionModel(self.sa).create_default_permissions(user=User.DEFAULT_USER)
 
     @staticmethod
     def check_waitress():
         """
         Function executed at the end of setup
         """
         if not __py_version__ >= (2, 6):
             notify('Python2.5 detected, please switch '
                    'egg:waitress#main -> egg:Paste#http '
                    'in your .ini file')

rhodecode/lib/dbmigrate/versions/012_version_1_7_0.py

@@ -1,62 +1,69 b''
 import logging
 import datetime
 
 from sqlalchemy import *
 from sqlalchemy.exc import DatabaseError
 from sqlalchemy.orm import relation, backref, class_mapper, joinedload
 from sqlalchemy.orm.session import Session
 from sqlalchemy.ext.declarative import declarative_base
 
 from rhodecode.lib.dbmigrate.migrate import *
 from rhodecode.lib.dbmigrate.migrate.changeset import *
 
 from rhodecode.model.meta import Base
 from rhodecode.model import meta
 from rhodecode.lib.dbmigrate.versions import _reset_base
 
 log = logging.getLogger(__name__)
 
 
 def upgrade(migrate_engine):
     """
     Upgrade operations go here.
     Don't create your own engine; bind migrate_engine to your metadata
     """
     _reset_base(migrate_engine)
 
     #==========================================================================
     # UserUserGroupToPerm
     #==========================================================================
     from rhodecode.lib.dbmigrate.schema.db_1_7_0 import UserUserGroupToPerm
     tbl = UserUserGroupToPerm.__table__
     tbl.create()
 
     #==========================================================================
     # UserGroupUserGroupToPerm
     #==========================================================================
     from rhodecode.lib.dbmigrate.schema.db_1_7_0 import UserGroupUserGroupToPerm
     tbl = UserGroupUserGroupToPerm.__table__
     tbl.create()
 
     #==========================================================================
+    # Gist
+    #==========================================================================
+    from rhodecode.lib.dbmigrate.schema.db_1_7_0 import Gist
+    tbl = Gist.__table__
+    tbl.create()
+
+    #==========================================================================
     # UserGroup
     #==========================================================================
     from rhodecode.lib.dbmigrate.schema.db_1_7_0 import UserGroup
     tbl = UserGroup.__table__
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=False, default=None)
     # create username column
     user_id.create(table=tbl)
 
     #==========================================================================
-    # UserGroup
+    # RepoGroup
     #==========================================================================
     from rhodecode.lib.dbmigrate.schema.db_1_7_0 import RepoGroup
     tbl = RepoGroup.__table__
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=False, default=None)
     # create username column
     user_id.create(table=tbl)
 
 
 def downgrade(migrate_engine):
     meta = MetaData()
     meta.bind = migrate_engine

rhodecode/lib/diffs.py

@@ -1,713 +1,711 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.diffs
     ~~~~~~~~~~~~~~~~~~~
 
     Set of diffing helpers, previously part of vcs
 
 
     :created_on: Dec 4, 2011
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :original copyright: 2007-2008 by Armin Ronacher
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import re
 import difflib
 import logging
 
 from itertools import tee, imap
 
 from pylons.i18n.translation import _
 
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.nodes import FileNode, SubModuleNode
 from rhodecode.lib.vcs.backends.base import EmptyChangeset
 from rhodecode.lib.helpers import escape
 from rhodecode.lib.utils2 import safe_unicode, safe_str
 
 log = logging.getLogger(__name__)
 
 
 def wrap_to_table(str_):
     return '''<table class="code-difftable">
                 <tr class="line no-comment">
                 <td class="lineno new"></td>
                 <td class="code no-comment"><pre>%s</pre></td>
                 </tr>
               </table>''' % str_
 
 
 def wrapped_diff(filenode_old, filenode_new, cut_off_limit=None,
                 ignore_whitespace=True, line_context=3,
                 enable_comments=False):
     """
     returns a wrapped diff into a table, checks for cut_off_limit and presents
     proper message
     """
 
     if filenode_old is None:
         filenode_old = FileNode(filenode_new.path, '', EmptyChangeset())
 
     if filenode_old.is_binary or filenode_new.is_binary:
         diff = wrap_to_table(_('Binary file'))
         stats = (0, 0)
         size = 0
 
     elif cut_off_limit != -1 and (cut_off_limit is None or
     (filenode_old.size < cut_off_limit and filenode_new.size < cut_off_limit)):
 
         f_gitdiff = get_gitdiff(filenode_old, filenode_new,
                                 ignore_whitespace=ignore_whitespace,
                                 context=line_context)
         diff_processor = DiffProcessor(f_gitdiff, format='gitdiff')
 
         diff = diff_processor.as_html(enable_comments=enable_comments)
         stats = diff_processor.stat()
         size = len(diff or '')
     else:
         diff = wrap_to_table(_('Changeset was too big and was cut off, use '
                                'diff menu to display this diff'))
         stats = (0, 0)
         size = 0
     if not diff:
         submodules = filter(lambda o: isinstance(o, SubModuleNode),
                             [filenode_new, filenode_old])
         if submodules:
             diff = wrap_to_table(escape('Submodule %r' % submodules[0]))
         else:
             diff = wrap_to_table(_('No changes detected'))
 
     cs1 = filenode_old.changeset.raw_id
     cs2 = filenode_new.changeset.raw_id
 
     return size, cs1, cs2, diff, stats
 
 
 def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True, context=3):
     """
     Returns git style diff between given ``filenode_old`` and ``filenode_new``.
 
     :param ignore_whitespace: ignore whitespaces in diff
     """
     # make sure we pass in default context
     context = context or 3
     submodules = filter(lambda o: isinstance(o, SubModuleNode),
                         [filenode_new, filenode_old])
     if submodules:
         return ''
 
     for filenode in (filenode_old, filenode_new):
         if not isinstance(filenode, FileNode):
             raise VCSError("Given object should be FileNode object, not %s"
                 % filenode.__class__)
 
     repo = filenode_new.changeset.repository
     old_raw_id = getattr(filenode_old.changeset, 'raw_id', repo.EMPTY_CHANGESET)
     new_raw_id = getattr(filenode_new.changeset, 'raw_id', repo.EMPTY_CHANGESET)
 
     vcs_gitdiff = repo.get_diff(old_raw_id, new_raw_id, filenode_new.path,
                                 ignore_whitespace, context)
     return vcs_gitdiff
 
 NEW_FILENODE = 1
 DEL_FILENODE = 2
 MOD_FILENODE = 3
 RENAMED_FILENODE = 4
 CHMOD_FILENODE = 5
 BIN_FILENODE = 6
 
 
 class DiffLimitExceeded(Exception):
     pass
 
 
 class LimitedDiffContainer(object):
 
     def __init__(self, diff_limit, cur_diff_size, diff):
         self.diff = diff
         self.diff_limit = diff_limit
         self.cur_diff_size = cur_diff_size
 
     def __iter__(self):
         for l in self.diff:
             yield l
 
 
 class DiffProcessor(object):
     """
     Give it a unified or git diff and it returns a list of the files that were
     mentioned in the diff together with a dict of meta information that
     can be used to render it in a HTML template.
     """
     _chunk_re = re.compile(r'^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')
     _newline_marker = re.compile(r'^\\ No newline at end of file')
     _git_header_re = re.compile(r"""
         #^diff[ ]--git
             [ ]a/(?P<a_path>.+?)[ ]b/(?P<b_path>.+?)\n
         (?:^similarity[ ]index[ ](?P<similarity_index>\d+)%\n
            ^rename[ ]from[ ](?P<rename_from>\S+)\n
            ^rename[ ]to[ ](?P<rename_to>\S+)(?:\n|$))?
         (?:^old[ ]mode[ ](?P<old_mode>\d+)\n
            ^new[ ]mode[ ](?P<new_mode>\d+)(?:\n|$))?
         (?:^new[ ]file[ ]mode[ ](?P<new_file_mode>.+)(?:\n|$))?
         (?:^deleted[ ]file[ ]mode[ ](?P<deleted_file_mode>.+)(?:\n|$))?
         (?:^index[ ](?P<a_blob_id>[0-9A-Fa-f]+)
             \.\.(?P<b_blob_id>[0-9A-Fa-f]+)[ ]?(?P<b_mode>.+)?(?:\n|$))?
         (?:^(?P<bin_patch>GIT[ ]binary[ ]patch)(?:\n|$))?
         (?:^---[ ](a/(?P<a_file>.+)|/dev/null)(?:\n|$))?
         (?:^\+\+\+[ ](b/(?P<b_file>.+)|/dev/null)(?:\n|$))?
     """, re.VERBOSE | re.MULTILINE)
     _hg_header_re = re.compile(r"""
         #^diff[ ]--git
             [ ]a/(?P<a_path>.+?)[ ]b/(?P<b_path>.+?)\n
         (?:^old[ ]mode[ ](?P<old_mode>\d+)\n
            ^new[ ]mode[ ](?P<new_mode>\d+)(?:\n|$))?
         (?:^similarity[ ]index[ ](?P<similarity_index>\d+)%(?:\n|$))?
         (?:^rename[ ]from[ ](?P<rename_from>\S+)\n
            ^rename[ ]to[ ](?P<rename_to>\S+)(?:\n|$))?
         (?:^new[ ]file[ ]mode[ ](?P<new_file_mode>.+)(?:\n|$))?
         (?:^deleted[ ]file[ ]mode[ ](?P<deleted_file_mode>.+)(?:\n|$))?
         (?:^index[ ](?P<a_blob_id>[0-9A-Fa-f]+)
             \.\.(?P<b_blob_id>[0-9A-Fa-f]+)[ ]?(?P<b_mode>.+)?(?:\n|$))?
         (?:^(?P<bin_patch>GIT[ ]binary[ ]patch)(?:\n|$))?
         (?:^---[ ](a/(?P<a_file>.+)|/dev/null)(?:\n|$))?
         (?:^\+\+\+[ ](b/(?P<b_file>.+)|/dev/null)(?:\n|$))?
     """, re.VERBOSE | re.MULTILINE)
 
     #used for inline highlighter word split
     _token_re = re.compile(r'()(&gt;|&lt;|&amp;|\W+?)')
 
     def __init__(self, diff, vcs='hg', format='gitdiff', diff_limit=None):
         """
         :param diff:   a text in diff format
         :param vcs: type of version controll hg or git
         :param format: format of diff passed, `udiff` or `gitdiff`
         :param diff_limit: define the size of diff that is considered "big"
             based on that parameter cut off will be triggered, set to None
             to show full diff
         """
         if not isinstance(diff, basestring):
             raise Exception('Diff must be a basestring got %s instead' % type(diff))
 
         self._diff = diff
         self._format = format
         self.adds = 0
         self.removes = 0
         # calculate diff size
         self.diff_size = len(diff)
         self.diff_limit = diff_limit
         self.cur_diff_size = 0
         self.parsed = False
         self.parsed_diff = []
         self.vcs = vcs
 
         if format == 'gitdiff':
             self.differ = self._highlight_line_difflib
             self._parser = self._parse_gitdiff
         else:
             self.differ = self._highlight_line_udiff
             self._parser = self._parse_udiff
 
     def _copy_iterator(self):
         """
         make a fresh copy of generator, we should not iterate thru
         an original as it's needed for repeating operations on
         this instance of DiffProcessor
         """
         self.__udiff, iterator_copy = tee(self.__udiff)
         return iterator_copy
 
     def _escaper(self, string):
         """
         Escaper for diff escapes special chars and checks the diff limit
 
         :param string:
-        :type string:
         """
 
         self.cur_diff_size += len(string)
 
         # escaper get's iterated on each .next() call and it checks if each
         # parsed line doesn't exceed the diff limit
         if self.diff_limit is not None and self.cur_diff_size > self.diff_limit:
             raise DiffLimitExceeded('Diff Limit Exceeded')
 
         return safe_unicode(string).replace('&', '&amp;')\
                 .replace('<', '&lt;')\
                 .replace('>', '&gt;')
 
     def _line_counter(self, l):
         """
         Checks each line and bumps total adds/removes for this diff
 
         :param l:
         """
         if l.startswith('+') and not l.startswith('+++'):
             self.adds += 1
         elif l.startswith('-') and not l.startswith('---'):
             self.removes += 1
         return safe_unicode(l)
 
     def _highlight_line_difflib(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
 
         if line['action'] == 'del':
             old, new = line, next_
         else:
             old, new = next_, line
 
         oldwords = self._token_re.split(old['line'])
         newwords = self._token_re.split(new['line'])
         sequence = difflib.SequenceMatcher(None, oldwords, newwords)
 
         oldfragments, newfragments = [], []
         for tag, i1, i2, j1, j2 in sequence.get_opcodes():
             oldfrag = ''.join(oldwords[i1:i2])
             newfrag = ''.join(newwords[j1:j2])
             if tag != 'equal':
                 if oldfrag:
                     oldfrag = '<del>%s</del>' % oldfrag
                 if newfrag:
                     newfrag = '<ins>%s</ins>' % newfrag
             oldfragments.append(oldfrag)
             newfragments.append(newfrag)
 
         old['line'] = "".join(oldfragments)
         new['line'] = "".join(newfragments)
 
     def _highlight_line_udiff(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
         start = 0
         limit = min(len(line['line']), len(next_['line']))
         while start < limit and line['line'][start] == next_['line'][start]:
             start += 1
         end = -1
         limit -= start
         while -end <= limit and line['line'][end] == next_['line'][end]:
             end -= 1
         end += 1
         if start or end:
             def do(l):
                 last = end + len(l['line'])
                 if l['action'] == 'add':
                     tag = 'ins'
                 else:
                     tag = 'del'
                 l['line'] = '%s<%s>%s</%s>%s' % (
                     l['line'][:start],
                     tag,
                     l['line'][start:last],
                     tag,
                     l['line'][last:]
                 )
             do(line)
             do(next_)
 
     def _get_header(self, diff_chunk):
         """
         parses the diff header, and returns parts, and leftover diff
         parts consists of 14 elements::
 
             a_path, b_path, similarity_index, rename_from, rename_to,
             old_mode, new_mode, new_file_mode, deleted_file_mode,
             a_blob_id, b_blob_id, b_mode, a_file, b_file
 
         :param diff_chunk:
-        :type diff_chunk:
         """
 
         if self.vcs == 'git':
             match = self._git_header_re.match(diff_chunk)
             diff = diff_chunk[match.end():]
             return match.groupdict(), imap(self._escaper, diff.splitlines(1))
         elif self.vcs == 'hg':
             match = self._hg_header_re.match(diff_chunk)
             diff = diff_chunk[match.end():]
             return match.groupdict(), imap(self._escaper, diff.splitlines(1))
         else:
             raise Exception('VCS type %s is not supported' % self.vcs)
 
     def _clean_line(self, line, command):
         if command in ['+', '-', ' ']:
             #only modify the line if it's actually a diff thing
             line = line[1:]
         return line
 
     def _parse_gitdiff(self, inline_diff=True):
         _files = []
         diff_container = lambda arg: arg
 
         ##split the diff in chunks of separate --git a/file b/file chunks
         for raw_diff in ('\n' + self._diff).split('\ndiff --git')[1:]:
             head, diff = self._get_header(raw_diff)
 
             op = None
             stats = {
                 'added': 0,
                 'deleted': 0,
                 'binary': False,
                 'ops': {},
             }
 
             if head['deleted_file_mode']:
                 op = 'D'
                 stats['binary'] = True
                 stats['ops'][DEL_FILENODE] = 'deleted file'
 
             elif head['new_file_mode']:
                 op = 'A'
                 stats['binary'] = True
                 stats['ops'][NEW_FILENODE] = 'new file %s' % head['new_file_mode']
             else:  # modify operation, can be cp, rename, chmod
                 # CHMOD
                 if head['new_mode'] and head['old_mode']:
                     op = 'M'
                     stats['binary'] = True
                     stats['ops'][CHMOD_FILENODE] = ('modified file chmod %s => %s'
                                         % (head['old_mode'], head['new_mode']))
                 # RENAME
                 if (head['rename_from'] and head['rename_to']
                       and head['rename_from'] != head['rename_to']):
                     op = 'M'
                     stats['binary'] = True
                     stats['ops'][RENAMED_FILENODE] = ('file renamed from %s to %s'
                                     % (head['rename_from'], head['rename_to']))
 
                 # FALL BACK: detect missed old style add or remove
                 if op is None:
                     if not head['a_file'] and head['b_file']:
                         op = 'A'
                         stats['binary'] = True
                         stats['ops'][NEW_FILENODE] = 'new file'
 
                     elif head['a_file'] and not head['b_file']:
                         op = 'D'
                         stats['binary'] = True
                         stats['ops'][DEL_FILENODE] = 'deleted file'
 
                 # it's not ADD not DELETE
                 if op is None:
                     op = 'M'
                     stats['binary'] = True
                     stats['ops'][MOD_FILENODE] = 'modified file'
 
             # a real non-binary diff
             if head['a_file'] or head['b_file']:
                 try:
                     chunks, _stats = self._parse_lines(diff)
                     stats['binary'] = False
                     stats['added'] = _stats[0]
                     stats['deleted'] = _stats[1]
                     # explicit mark that it's a modified file
                     if op == 'M':
                         stats['ops'][MOD_FILENODE] = 'modified file'
 
                 except DiffLimitExceeded:
                     diff_container = lambda _diff: \
                         LimitedDiffContainer(self.diff_limit,
                                             self.cur_diff_size, _diff)
                     break
             else:  # GIT binary patch (or empty diff)
                 # GIT Binary patch
                 if head['bin_patch']:
                     stats['ops'][BIN_FILENODE] = 'binary diff not shown'
                 chunks = []
 
             chunks.insert(0, [{
                 'old_lineno': '',
                 'new_lineno': '',
                 'action':     'context',
                 'line':       msg,
                 } for _op, msg in stats['ops'].iteritems()
                   if _op not in [MOD_FILENODE]])
 
             _files.append({
                 'filename':         head['b_path'],
                 'old_revision':     head['a_blob_id'],
                 'new_revision':     head['b_blob_id'],
                 'chunks':           chunks,
                 'operation':        op,
                 'stats':            stats,
             })
 
         sorter = lambda info: {'A': 0, 'M': 1, 'D': 2}.get(info['operation'])
 
         if not inline_diff:
             return diff_container(sorted(_files, key=sorter))
 
         # highlight inline changes
         for diff_data in _files:
             for chunk in diff_data['chunks']:
                 lineiter = iter(chunk)
                 try:
                     while 1:
                         line = lineiter.next()
                         if line['action'] not in ['unmod', 'context']:
                             nextline = lineiter.next()
                             if nextline['action'] in ['unmod', 'context'] or \
                                nextline['action'] == line['action']:
                                 continue
                             self.differ(line, nextline)
                 except StopIteration:
                     pass
 
         return diff_container(sorted(_files, key=sorter))
 
     def _parse_udiff(self, inline_diff=True):
         raise NotImplementedError()
 
     def _parse_lines(self, diff):
         """
         Parse the diff an return data for the template.
         """
 
         lineiter = iter(diff)
         stats = [0, 0]
 
         try:
             chunks = []
             line = lineiter.next()
 
             while line:
                 lines = []
                 chunks.append(lines)
 
                 match = self._chunk_re.match(line)
 
                 if not match:
                     break
 
                 gr = match.groups()
                 (old_line, old_end,
                  new_line, new_end) = [int(x or 1) for x in gr[:-1]]
                 old_line -= 1
                 new_line -= 1
 
                 context = len(gr) == 5
                 old_end += old_line
                 new_end += new_line
 
                 if context:
                     # skip context only if it's first line
                     if int(gr[0]) > 1:
                         lines.append({
                             'old_lineno': '...',
                             'new_lineno': '...',
                             'action':     'context',
                             'line':       line,
                         })
 
                 line = lineiter.next()
 
                 while old_line < old_end or new_line < new_end:
                     command = ' '
                     if line:
                         command = line[0]
 
                     affects_old = affects_new = False
 
                     # ignore those if we don't expect them
                     if command in '#@':
                         continue
                     elif command == '+':
                         affects_new = True
                         action = 'add'
                         stats[0] += 1
                     elif command == '-':
                         affects_old = True
                         action = 'del'
                         stats[1] += 1
                     else:
                         affects_old = affects_new = True
                         action = 'unmod'
 
                     if not self._newline_marker.match(line):
                         old_line += affects_old
                         new_line += affects_new
                         lines.append({
                             'old_lineno':   affects_old and old_line or '',
                             'new_lineno':   affects_new and new_line or '',
                             'action':       action,
                             'line':         self._clean_line(line, command)
                         })
 
                     line = lineiter.next()
 
                     if self._newline_marker.match(line):
                         # we need to append to lines, since this is not
                         # counted in the line specs of diff
                         lines.append({
                             'old_lineno':   '...',
                             'new_lineno':   '...',
                             'action':       'context',
                             'line':         self._clean_line(line, command)
                         })
 
         except StopIteration:
             pass
         return chunks, stats
 
     def _safe_id(self, idstring):
         """Make a string safe for including in an id attribute.
 
         The HTML spec says that id attributes 'must begin with
         a letter ([A-Za-z]) and may be followed by any number
         of letters, digits ([0-9]), hyphens ("-"), underscores
         ("_"), colons (":"), and periods (".")'. These regexps
         are slightly over-zealous, in that they remove colons
         and periods unnecessarily.
 
         Whitespace is transformed into underscores, and then
         anything which is not a hyphen or a character that
         matches \w (alphanumerics and underscore) is removed.
 
         """
         # Transform all whitespace to underscore
         idstring = re.sub(r'\s', "_", '%s' % idstring)
         # Remove everything that is not a hyphen or a member of \w
         idstring = re.sub(r'(?!-)\W', "", idstring).lower()
         return idstring
 
     def prepare(self, inline_diff=True):
         """
         Prepare the passed udiff for HTML rendering. It'l return a list
         of dicts with diff information
         """
         parsed = self._parser(inline_diff=inline_diff)
         self.parsed = True
         self.parsed_diff = parsed
         return parsed
 
     def as_raw(self, diff_lines=None):
         """
         Returns raw string diff
         """
         return self._diff
         #return u''.join(imap(self._line_counter, self._diff.splitlines(1)))
 
     def as_html(self, table_class='code-difftable', line_class='line',
                 old_lineno_class='lineno old', new_lineno_class='lineno new',
                 code_class='code', enable_comments=False, parsed_lines=None):
         """
         Return given diff as html table with customized css classes
         """
         def _link_to_if(condition, label, url):
             """
             Generates a link if condition is meet or just the label if not.
             """
 
             if condition:
                 return '''<a href="%(url)s">%(label)s</a>''' % {
                     'url': url,
                     'label': label
                 }
             else:
                 return label
         if not self.parsed:
             self.prepare()
 
         diff_lines = self.parsed_diff
         if parsed_lines:
             diff_lines = parsed_lines
 
         _html_empty = True
         _html = []
         _html.append('''<table class="%(table_class)s">\n''' % {
             'table_class': table_class
         })
 
         for diff in diff_lines:
             for line in diff['chunks']:
                 _html_empty = False
                 for change in line:
                     _html.append('''<tr class="%(lc)s %(action)s">\n''' % {
                         'lc': line_class,
                         'action': change['action']
                     })
                     anchor_old_id = ''
                     anchor_new_id = ''
                     anchor_old = "%(filename)s_o%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['old_lineno']
                     }
                     anchor_new = "%(filename)s_n%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['new_lineno']
                     }
                     cond_old = (change['old_lineno'] != '...' and
                                 change['old_lineno'])
                     cond_new = (change['new_lineno'] != '...' and
                                 change['new_lineno'])
                     if cond_old:
                         anchor_old_id = 'id="%s"' % anchor_old
                     if cond_new:
                         anchor_new_id = 'id="%s"' % anchor_new
                     ###########################################################
                     # OLD LINE NUMBER
                     ###########################################################
                     _html.append('''\t<td %(a_id)s class="%(olc)s">''' % {
                         'a_id': anchor_old_id,
                         'olc': old_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(True, change['old_lineno'],
                                             '#%s' % anchor_old)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # NEW LINE NUMBER
                     ###########################################################
 
                     _html.append('''\t<td %(a_id)s class="%(nlc)s">''' % {
                         'a_id': anchor_new_id,
                         'nlc': new_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(True, change['new_lineno'],
                                             '#%s' % anchor_new)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # CODE
                     ###########################################################
                     comments = '' if enable_comments else 'no-comment'
                     _html.append('''\t<td class="%(cc)s %(inc)s">''' % {
                         'cc': code_class,
                         'inc': comments
                     })
                     _html.append('''\n\t\t<pre>%(code)s</pre>\n''' % {
                         'code': change['line']
                     })
 
                     _html.append('''\t</td>''')
                     _html.append('''\n</tr>\n''')
         _html.append('''</table>''')
         if _html_empty:
             return None
         return ''.join(_html)
 
     def stat(self):
         """
         Returns tuple of added, and removed lines for this instance
         """
         return self.adds, self.removes

rhodecode/lib/exceptions.py

@@ -1,84 +1,88 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.exceptions
     ~~~~~~~~~~~~~~~~~~~~~~~~
 
     Set of custom exceptions used in RhodeCode
 
     :created_on: Nov 17, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from webob.exc import HTTPClientError
 
 
 class LdapUsernameError(Exception):
     pass
 
 
 class LdapPasswordError(Exception):
     pass
 
 
 class LdapConnectionError(Exception):
     pass
 
 
 class LdapImportError(Exception):
     pass
 
 
 class DefaultUserException(Exception):
     pass
 
 
 class UserOwnsReposException(Exception):
     pass
 
 
 class UserGroupsAssignedException(Exception):
     pass
 
 
 class StatusChangeOnClosedPullRequestError(Exception):
     pass
 
 
 class AttachedForksError(Exception):
     pass
 
 
 class RepoGroupAssignmentError(Exception):
     pass
 
 
+class NonRelativePathError(Exception):
+    pass
+
+
 class HTTPLockedRC(HTTPClientError):
     """
     Special Exception For locked Repos in RhodeCode, the return code can
     be overwritten by _code keyword argument passed into constructors
     """
     code = 423
     title = explanation = 'Repository Locked'
 
     def __init__(self, reponame, username, *args, **kwargs):
         from rhodecode import CONFIG
         from rhodecode.lib.utils2 import safe_int
         _code = CONFIG.get('lock_ret_code')
         self.code = safe_int(_code, self.code)
         self.title = self.explanation = ('Repository `%s` locked by '
                                          'user `%s`' % (reponame, username))
         super(HTTPLockedRC, self).__init__(*args, **kwargs)

rhodecode/lib/hooks.py

@@ -1,390 +1,387 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.hooks
     ~~~~~~~~~~~~~~~~~~~
 
     Hooks runned by rhodecode
 
     :created_on: Aug 6, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import os
 import sys
 import time
 import binascii
 import traceback
 from inspect import isfunction
 
 from mercurial.scmutil import revrange
 from mercurial.node import nullrev
 
 from rhodecode.lib import helpers as h
 from rhodecode.lib.utils import action_logger
 from rhodecode.lib.vcs.backends.base import EmptyChangeset
 from rhodecode.lib.compat import json
 from rhodecode.lib.exceptions import HTTPLockedRC
 from rhodecode.lib.utils2 import safe_str, _extract_extras
 from rhodecode.model.db import Repository, User
 
 
 def _get_scm_size(alias, root_path):
 
     if not alias.startswith('.'):
         alias += '.'
 
     size_scm, size_root = 0, 0
     for path, dirs, files in os.walk(safe_str(root_path)):
         if path.find(alias) != -1:
             for f in files:
                 try:
                     size_scm += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
         else:
             for f in files:
                 try:
                     size_root += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
 
     size_scm_f = h.format_byte_size(size_scm)
     size_root_f = h.format_byte_size(size_root)
     size_total_f = h.format_byte_size(size_root + size_scm)
 
     return size_scm_f, size_root_f, size_total_f
 
 
 def repo_size(ui, repo, hooktype=None, **kwargs):
     """
     Presents size of repository after push
 
     :param ui:
     :param repo:
     :param hooktype:
     """
 
     size_hg_f, size_root_f, size_total_f = _get_scm_size('.hg', repo.root)
 
     last_cs = repo[len(repo) - 1]
 
     msg = ('Repository size .hg:%s repo:%s total:%s\n'
            'Last revision is now r%s:%s\n') % (
         size_hg_f, size_root_f, size_total_f, last_cs.rev(), last_cs.hex()[:12]
     )
 
     sys.stdout.write(msg)
 
 
 def pre_push(ui, repo, **kwargs):
     # pre push function, currently used to ban pushing when
     # repository is locked
     ex = _extract_extras()
 
     usr = User.get_by_username(ex.username)
     if ex.locked_by[0] and usr.user_id != int(ex.locked_by[0]):
         locked_by = User.get(ex.locked_by[0]).username
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(ex.repository, locked_by)
         if str(_http_ret.code).startswith('2'):
             #2xx Codes don't raise exceptions
             sys.stdout.write(_http_ret.title)
         else:
             raise _http_ret
 
 
 def pre_pull(ui, repo, **kwargs):
     # pre push function, currently used to ban pushing when
     # repository is locked
     ex = _extract_extras()
     if ex.locked_by[0]:
         locked_by = User.get(ex.locked_by[0]).username
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(ex.repository, locked_by)
         if str(_http_ret.code).startswith('2'):
             #2xx Codes don't raise exceptions
             sys.stdout.write(_http_ret.title)
         else:
             raise _http_ret
 
 
 def log_pull_action(ui, repo, **kwargs):
     """
     Logs user last pull action
 
     :param ui:
     :param repo:
     """
     ex = _extract_extras()
 
     user = User.get_by_username(ex.username)
     action = 'pull'
     action_logger(user, action, ex.repository, ex.ip, commit=True)
     # extension hook call
     from rhodecode import EXTENSIONS
     callback = getattr(EXTENSIONS, 'PULL_HOOK', None)
     if isfunction(callback):
         kw = {}
         kw.update(ex)
         callback(**kw)
 
     if ex.make_lock is not None and ex.make_lock:
         Repository.lock(Repository.get_by_repo_name(ex.repository), user.user_id)
         #msg = 'Made lock on repo `%s`' % repository
         #sys.stdout.write(msg)
 
     if ex.locked_by[0]:
         locked_by = User.get(ex.locked_by[0]).username
         _http_ret = HTTPLockedRC(ex.repository, locked_by)
         if str(_http_ret.code).startswith('2'):
             #2xx Codes don't raise exceptions
             sys.stdout.write(_http_ret.title)
     return 0
 
 
 def log_push_action(ui, repo, **kwargs):
     """
     Maps user last push action to new changeset id, from mercurial
 
     :param ui:
     :param repo: repo object containing the `ui` object
     """
 
     ex = _extract_extras()
 
     action = ex.action + ':%s'
 
     if ex.scm == 'hg':
         node = kwargs['node']
 
         def get_revs(repo, rev_opt):
             if rev_opt:
                 revs = revrange(repo, rev_opt)
 
                 if len(revs) == 0:
                     return (nullrev, nullrev)
                 return (max(revs), min(revs))
             else:
                 return (len(repo) - 1, 0)
 
         stop, start = get_revs(repo, [node + ':'])
         h = binascii.hexlify
         revs = [h(repo[r].node()) for r in xrange(start, stop + 1)]
     elif ex.scm == 'git':
         revs = kwargs.get('_git_revs', [])
         if '_git_revs' in kwargs:
             kwargs.pop('_git_revs')
 
     action = action % ','.join(revs)
 
     action_logger(ex.username, action, ex.repository, ex.ip, commit=True)
 
     # extension hook call
     from rhodecode import EXTENSIONS
     callback = getattr(EXTENSIONS, 'PUSH_HOOK', None)
     if isfunction(callback):
         kw = {'pushed_revs': revs}
         kw.update(ex)
         callback(**kw)
 
     if ex.make_lock is not None and not ex.make_lock:
         Repository.unlock(Repository.get_by_repo_name(ex.repository))
         msg = 'Released lock on repo `%s`\n' % ex.repository
         sys.stdout.write(msg)
 
     if ex.locked_by[0]:
         locked_by = User.get(ex.locked_by[0]).username
         _http_ret = HTTPLockedRC(ex.repository, locked_by)
         if str(_http_ret.code).startswith('2'):
             #2xx Codes don't raise exceptions
             sys.stdout.write(_http_ret.title)
 
     return 0
 
 
 def log_create_repository(repository_dict, created_by, **kwargs):
     """
     Post create repository Hook. This is a dummy function for admins to re-use
     if needed. It's taken from rhodecode-extensions module and executed
     if present
 
     :param repository: dict dump of repository object
     :param created_by: username who created repository
 
     available keys of repository_dict:
 
      'repo_type',
      'description',
      'private',
      'created_on',
      'enable_downloads',
      'repo_id',
      'user_id',
      'enable_statistics',
      'clone_uri',
      'fork_id',
      'group_id',
      'repo_name'
 
     """
     from rhodecode import EXTENSIONS
     callback = getattr(EXTENSIONS, 'CREATE_REPO_HOOK', None)
     if isfunction(callback):
         kw = {}
         kw.update(repository_dict)
         kw.update({'created_by': created_by})
         kw.update(kwargs)
         return callback(**kw)
 
     return 0
 
 
 def log_delete_repository(repository_dict, deleted_by, **kwargs):
     """
     Post delete repository Hook. This is a dummy function for admins to re-use
     if needed. It's taken from rhodecode-extensions module and executed
     if present
 
     :param repository: dict dump of repository object
     :param deleted_by: username who deleted the repository
 
     available keys of repository_dict:
 
      'repo_type',
      'description',
      'private',
      'created_on',
      'enable_downloads',
      'repo_id',
      'user_id',
      'enable_statistics',
      'clone_uri',
      'fork_id',
      'group_id',
      'repo_name'
 
     """
     from rhodecode import EXTENSIONS
     callback = getattr(EXTENSIONS, 'DELETE_REPO_HOOK', None)
     if isfunction(callback):
         kw = {}
         kw.update(repository_dict)
         kw.update({'deleted_by': deleted_by,
                    'deleted_on': time.time()})
         kw.update(kwargs)
         return callback(**kw)
 
     return 0
 
 
 handle_git_pre_receive = (lambda repo_path, revs, env:
     handle_git_receive(repo_path, revs, env, hook_type='pre'))
 handle_git_post_receive = (lambda repo_path, revs, env:
     handle_git_receive(repo_path, revs, env, hook_type='post'))
 
 
 def handle_git_receive(repo_path, revs, env, hook_type='post'):
     """
     A really hacky method that is runned by git post-receive hook and logs
     an push action together with pushed revisions. It's executed by subprocess
     thus needs all info to be able to create a on the fly pylons enviroment,
     connect to database and run the logging code. Hacky as sh*t but works.
 
     :param repo_path:
-    :type repo_path:
     :param revs:
-    :type revs:
     :param env:
-    :type env:
     """
     from paste.deploy import appconfig
     from sqlalchemy import engine_from_config
     from rhodecode.config.environment import load_environment
     from rhodecode.model import init_model
     from rhodecode.model.db import RhodeCodeUi
     from rhodecode.lib.utils import make_ui
     extras = _extract_extras(env)
 
     path, ini_name = os.path.split(extras['config'])
     conf = appconfig('config:%s' % ini_name, relative_to=path)
     load_environment(conf.global_conf, conf.local_conf)
 
     engine = engine_from_config(conf, 'sqlalchemy.db1.')
     init_model(engine)
 
     baseui = make_ui('db')
     # fix if it's not a bare repo
     if repo_path.endswith(os.sep + '.git'):
         repo_path = repo_path[:-5]
 
     repo = Repository.get_by_full_path(repo_path)
     if not repo:
         raise OSError('Repository %s not found in database'
                       % (safe_str(repo_path)))
 
     _hooks = dict(baseui.configitems('hooks')) or {}
 
     if hook_type == 'pre':
         repo = repo.scm_instance
     else:
         #post push shouldn't use the cached instance never
         repo = repo.scm_instance_no_cache()
 
     if hook_type == 'pre':
         pre_push(baseui, repo)
 
     # if push hook is enabled via web interface
     elif hook_type == 'post' and _hooks.get(RhodeCodeUi.HOOK_PUSH):
 
         rev_data = []
         for l in revs:
             old_rev, new_rev, ref = l.split(' ')
             _ref_data = ref.split('/')
             if _ref_data[1] in ['tags', 'heads']:
                 rev_data.append({'old_rev': old_rev,
                                  'new_rev': new_rev,
                                  'ref': ref,
                                  'type': _ref_data[1],
                                  'name': _ref_data[2].strip()})
 
         git_revs = []
         for push_ref  in rev_data:
             _type = push_ref['type']
             if _type == 'heads':
                 if push_ref['old_rev'] == EmptyChangeset().raw_id:
                     cmd = "for-each-ref --format='%(refname)' 'refs/heads/*'"
                     heads = repo.run_git_command(cmd)[0]
                     heads = heads.replace(push_ref['ref'], '')
                     heads = ' '.join(map(lambda c: c.strip('\n').strip(),
                                          heads.splitlines()))
                     cmd = (('log %(new_rev)s' % push_ref) +
                            ' --reverse --pretty=format:"%H" --not ' + heads)
                     git_revs += repo.run_git_command(cmd)[0].splitlines()
 
                 elif push_ref['new_rev'] == EmptyChangeset().raw_id:
                     #delete branch case
                     git_revs += ['delete_branch=>%s' % push_ref['name']]
                 else:
                     cmd = (('log %(old_rev)s..%(new_rev)s' % push_ref) +
                            ' --reverse --pretty=format:"%H"')
                     git_revs += repo.run_git_command(cmd)[0].splitlines()
 
             elif _type == 'tags':
                 git_revs += ['tag=>%s' % push_ref['name']]
 
         log_push_action(baseui, repo, _git_revs=git_revs)

rhodecode/lib/middleware/pygrack.py

@@ -1,201 +1,200 b''
 import os
 import socket
 import logging
 import subprocess
 import traceback
 
 from webob import Request, Response, exc
 
 import rhodecode
 from rhodecode.lib.vcs import subprocessio
 
 log = logging.getLogger(__name__)
 
 
 class FileWrapper(object):
 
     def __init__(self, fd, content_length):
         self.fd = fd
         self.content_length = content_length
         self.remain = content_length
 
     def read(self, size):
         if size <= self.remain:
             try:
                 data = self.fd.read(size)
             except socket.error:
                 raise IOError(self)
             self.remain -= size
         elif self.remain:
             data = self.fd.read(self.remain)
             self.remain = 0
         else:
             data = None
         return data
 
     def __repr__(self):
         return '<FileWrapper %s len: %s, read: %s>' % (
             self.fd, self.content_length, self.content_length - self.remain
         )
 
 
 class GitRepository(object):
     git_folder_signature = set(['config', 'head', 'info', 'objects', 'refs'])
     commands = ['git-upload-pack', 'git-receive-pack']
 
     def __init__(self, repo_name, content_path, extras):
         files = set([f.lower() for f in os.listdir(content_path)])
         if  not (self.git_folder_signature.intersection(files)
                 == self.git_folder_signature):
             raise OSError('%s missing git signature' % content_path)
         self.content_path = content_path
         self.valid_accepts = ['application/x-%s-result' %
                               c for c in self.commands]
         self.repo_name = repo_name
         self.extras = extras
 
     def _get_fixedpath(self, path):
         """
         Small fix for repo_path
 
         :param path:
-        :type path:
         """
         return path.split(self.repo_name, 1)[-1].strip('/')
 
     def inforefs(self, request, environ):
         """
         WSGI Response producer for HTTP GET Git Smart
         HTTP /info/refs request.
         """
 
         git_command = request.GET.get('service')
         if git_command not in self.commands:
             log.debug('command %s not allowed' % git_command)
             return exc.HTTPMethodNotAllowed()
 
         # note to self:
         # please, resist the urge to add '\n' to git capture and increment
         # line count by 1.
         # The code in Git client not only does NOT need '\n', but actually
         # blows up if you sprinkle "flush" (0000) as "0001\n".
         # It reads binary, per number of bytes specified.
         # if you do add '\n' as part of data, count it.
         server_advert = '# service=%s' % git_command
         packet_len = str(hex(len(server_advert) + 4)[2:].rjust(4, '0')).lower()
         _git_path = rhodecode.CONFIG.get('git_path', 'git')
         try:
             out = subprocessio.SubprocessIOChunker(
                 r'%s %s --stateless-rpc --advertise-refs "%s"' % (
                             _git_path, git_command[4:], self.content_path),
                 starting_values=[
                     packet_len + server_advert + '0000'
                 ]
             )
         except EnvironmentError, e:
             log.error(traceback.format_exc())
             raise exc.HTTPExpectationFailed()
         resp = Response()
         resp.content_type = 'application/x-%s-advertisement' % str(git_command)
         resp.charset = None
         resp.app_iter = out
         return resp
 
     def backend(self, request, environ):
         """
         WSGI Response producer for HTTP POST Git Smart HTTP requests.
         Reads commands and data from HTTP POST's body.
         returns an iterator obj with contents of git command's
         response to stdout
         """
         git_command = self._get_fixedpath(request.path_info)
         if git_command not in self.commands:
             log.debug('command %s not allowed' % git_command)
             return exc.HTTPMethodNotAllowed()
 
         if 'CONTENT_LENGTH' in environ:
             inputstream = FileWrapper(environ['wsgi.input'],
                                       request.content_length)
         else:
             inputstream = environ['wsgi.input']
 
         try:
             gitenv = os.environ
             # forget all configs
             gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
             opts = dict(
                 env=gitenv,
                 cwd=os.getcwd()
             )
             cmd = r'git %s --stateless-rpc "%s"' % (git_command[4:],
                                                     self.content_path),
             log.debug('handling cmd %s' % cmd)
             out = subprocessio.SubprocessIOChunker(
                 cmd,
                 inputstream=inputstream,
                 **opts
             )
         except EnvironmentError, e:
             log.error(traceback.format_exc())
             raise exc.HTTPExpectationFailed()
 
         if git_command in [u'git-receive-pack']:
             # updating refs manually after each push.
             # Needed for pre-1.7.0.4 git clients using regular HTTP mode.
             _git_path = rhodecode.CONFIG.get('git_path', 'git')
             cmd = (u'%s --git-dir "%s" '
                     'update-server-info' % (_git_path, self.content_path))
             log.debug('handling cmd %s' % cmd)
             subprocess.call(cmd, shell=True)
 
         resp = Response()
         resp.content_type = 'application/x-%s-result' % git_command.encode('utf8')
         resp.charset = None
         resp.app_iter = out
         return resp
 
     def __call__(self, environ, start_response):
         request = Request(environ)
         _path = self._get_fixedpath(request.path_info)
         if _path.startswith('info/refs'):
             app = self.inforefs
         elif [a for a in self.valid_accepts if a in request.accept]:
             app = self.backend
         try:
             resp = app(request, environ)
         except exc.HTTPException, e:
             resp = e
             log.error(traceback.format_exc())
         except Exception, e:
             log.error(traceback.format_exc())
             resp = exc.HTTPInternalServerError()
         return resp(environ, start_response)
 
 
 class GitDirectory(object):
 
     def __init__(self, repo_root, repo_name, extras):
         repo_location = os.path.join(repo_root, repo_name)
         if not os.path.isdir(repo_location):
             raise OSError(repo_location)
 
         self.content_path = repo_location
         self.repo_name = repo_name
         self.repo_location = repo_location
         self.extras = extras
 
     def __call__(self, environ, start_response):
         content_path = self.content_path
         try:
             app = GitRepository(self.repo_name, content_path, self.extras)
         except (AssertionError, OSError):
             content_path = os.path.join(content_path, '.git')
             if os.path.isdir(content_path):
                 app = GitRepository(self.repo_name, content_path, self.extras)
             else:
                 return exc.HTTPNotFound()(environ, start_response)
         return app(environ, start_response)
 
 
 def make_wsgi_app(repo_name, repo_root, extras):
     return GitDirectory(repo_root, repo_name, extras)

rhodecode/lib/utils2.py

@@ -1,609 +1,646 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.utils
     ~~~~~~~~~~~~~~~~~~~
 
     Some simple helper functions
 
     :created_on: Jan 5, 2011
     :author: marcink
     :copyright: (C) 2011-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import re
 import sys
 import time
+import uuid
 import datetime
 import traceback
 import webob
 
 from pylons.i18n.translation import _, ungettext
 from rhodecode.lib.vcs.utils.lazy import LazyProperty
 from rhodecode.lib.compat import json
 
 
 def __get_lem():
     """
     Get language extension map based on what's inside pygments lexers
     """
     from pygments import lexers
     from string import lower
     from collections import defaultdict
 
     d = defaultdict(lambda: [])
 
     def __clean(s):
         s = s.lstrip('*')
         s = s.lstrip('.')
 
         if s.find('[') != -1:
             exts = []
             start, stop = s.find('['), s.find(']')
 
             for suffix in s[start + 1:stop]:
                 exts.append(s[:s.find('[')] + suffix)
             return map(lower, exts)
         else:
             return map(lower, [s])
 
     for lx, t in sorted(lexers.LEXERS.items()):
         m = map(__clean, t[-2])
         if m:
             m = reduce(lambda x, y: x + y, m)
             for ext in m:
                 desc = lx.replace('Lexer', '')
                 d[ext].append(desc)
 
     return dict(d)
 
 
 def str2bool(_str):
     """
     returs True/False value from given string, it tries to translate the
     string into boolean
 
     :param _str: string value to translate into boolean
     :rtype: boolean
     :returns: boolean from given string
     """
     if _str is None:
         return False
     if _str in (True, False):
         return _str
     _str = str(_str).strip().lower()
     return _str in ('t', 'true', 'y', 'yes', 'on', '1')
 
 
 def aslist(obj, sep=None, strip=True):
     """
     Returns given string separated by sep as list
 
     :param obj:
     :param sep:
     :param strip:
     """
     if isinstance(obj, (basestring)):
         lst = obj.split(sep)
         if strip:
             lst = [v.strip() for v in lst]
         return lst
     elif isinstance(obj, (list, tuple)):
         return obj
     elif obj is None:
         return []
     else:
         return [obj]
 
 
 def convert_line_endings(line, mode):
     """
     Converts a given line  "line end" accordingly to given mode
 
     Available modes are::
         0 - Unix
         1 - Mac
         2 - DOS
 
     :param line: given line to convert
     :param mode: mode to convert to
     :rtype: str
     :return: converted line according to mode
     """
     from string import replace
 
     if mode == 0:
             line = replace(line, '\r\n', '\n')
             line = replace(line, '\r', '\n')
     elif mode == 1:
             line = replace(line, '\r\n', '\r')
             line = replace(line, '\n', '\r')
     elif mode == 2:
             line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line)
     return line
 
 
 def detect_mode(line, default):
     """
     Detects line break for given line, if line break couldn't be found
     given default value is returned
 
     :param line: str line
     :param default: default
     :rtype: int
     :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
     """
     if line.endswith('\r\n'):
         return 2
     elif line.endswith('\n'):
         return 0
     elif line.endswith('\r'):
         return 1
     else:
         return default
 
 
 def generate_api_key(username, salt=None):
     """
     Generates unique API key for given username, if salt is not given
     it'll be generated from some random string
 
     :param username: username as string
     :param salt: salt to hash generate KEY
     :rtype: str
     :returns: sha1 hash from username+salt
     """
     from tempfile import _RandomNameSequence
     import hashlib
 
     if salt is None:
         salt = _RandomNameSequence().next()
 
     return hashlib.sha1(username + salt).hexdigest()
 
 
 def safe_int(val, default=None):
     """
     Returns int() of val if val is not convertable to int use default
     instead
 
     :param val:
     :param default:
     """
 
     try:
         val = int(val)
     except (ValueError, TypeError):
         val = default
 
     return val
 
 
 def safe_unicode(str_, from_encoding=None):
     """
     safe unicode function. Does few trick to turn str_ into unicode
 
     In case of UnicodeDecode error we try to return it with encoding detected
     by chardet library if it fails fallback to unicode with errors replaced
 
     :param str_: string to decode
     :rtype: unicode
     :returns: unicode object
     """
     if isinstance(str_, unicode):
         return str_
 
     if not from_encoding:
         import rhodecode
         DEFAULT_ENCODINGS = aslist(rhodecode.CONFIG.get('default_encoding',
                                                         'utf8'), sep=',')
         from_encoding = DEFAULT_ENCODINGS
 
     if not isinstance(from_encoding, (list, tuple)):
         from_encoding = [from_encoding]
 
     try:
         return unicode(str_)
     except UnicodeDecodeError:
         pass
 
     for enc in from_encoding:
         try:
             return unicode(str_, enc)
         except UnicodeDecodeError:
             pass
 
     try:
         import chardet
         encoding = chardet.detect(str_)['encoding']
         if encoding is None:
             raise Exception()
         return str_.decode(encoding)
     except (ImportError, UnicodeDecodeError, Exception):
         return unicode(str_, from_encoding[0], 'replace')
 
 
 def safe_str(unicode_, to_encoding=None):
     """
     safe str function. Does few trick to turn unicode_ into string
 
     In case of UnicodeEncodeError we try to return it with encoding detected
     by chardet library if it fails fallback to string with errors replaced
 
     :param unicode_: unicode to encode
     :rtype: str
     :returns: str object
     """
 
     # if it's not basestr cast to str
     if not isinstance(unicode_, basestring):
         return str(unicode_)
 
     if isinstance(unicode_, str):
         return unicode_
 
     if not to_encoding:
         import rhodecode
         DEFAULT_ENCODINGS = aslist(rhodecode.CONFIG.get('default_encoding',
                                                         'utf8'), sep=',')
         to_encoding = DEFAULT_ENCODINGS
 
     if not isinstance(to_encoding, (list, tuple)):
         to_encoding = [to_encoding]
 
     for enc in to_encoding:
         try:
             return unicode_.encode(enc)
         except UnicodeEncodeError:
             pass
 
     try:
         import chardet
         encoding = chardet.detect(unicode_)['encoding']
         if encoding is None:
             raise UnicodeEncodeError()
 
         return unicode_.encode(encoding)
     except (ImportError, UnicodeEncodeError):
         return unicode_.encode(to_encoding[0], 'replace')
 
     return safe_str
 
 
 def remove_suffix(s, suffix):
     if s.endswith(suffix):
         s = s[:-1 * len(suffix)]
     return s
 
 
 def remove_prefix(s, prefix):
     if s.startswith(prefix):
         s = s[len(prefix):]
     return s
 
 
 def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
     """
     Custom engine_from_config functions that makes sure we use NullPool for
     file based sqlite databases. This prevents errors on sqlite. This only
     applies to sqlalchemy versions < 0.7.0
 
     """
     import sqlalchemy
     from sqlalchemy import engine_from_config as efc
     import logging
 
     if int(sqlalchemy.__version__.split('.')[1]) < 7:
 
         # This solution should work for sqlalchemy < 0.7.0, and should use
         # proxy=TimerProxy() for execution time profiling
 
         from sqlalchemy.pool import NullPool
         url = configuration[prefix + 'url']
 
         if url.startswith('sqlite'):
             kwargs.update({'poolclass': NullPool})
         return efc(configuration, prefix, **kwargs)
     else:
         import time
         from sqlalchemy import event
         from sqlalchemy.engine import Engine
 
         log = logging.getLogger('sqlalchemy.engine')
         BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38)
         engine = efc(configuration, prefix, **kwargs)
 
         def color_sql(sql):
             COLOR_SEQ = "\033[1;%dm"
             COLOR_SQL = YELLOW
             normal = '\x1b[0m'
             return ''.join([COLOR_SEQ % COLOR_SQL, sql, normal])
 
         if configuration['debug']:
             #attach events only for debug configuration
 
             def before_cursor_execute(conn, cursor, statement,
                                     parameters, context, executemany):
                 context._query_start_time = time.time()
                 log.info(color_sql(">>>>> STARTING QUERY >>>>>"))
 
             def after_cursor_execute(conn, cursor, statement,
                                     parameters, context, executemany):
                 total = time.time() - context._query_start_time
                 log.info(color_sql("<<<<< TOTAL TIME: %f <<<<<" % total))
 
             event.listen(engine, "before_cursor_execute",
                          before_cursor_execute)
             event.listen(engine, "after_cursor_execute",
                          after_cursor_execute)
 
     return engine
 
 
 def age(prevdate, show_short_version=False, now=None):
     """
     turns a datetime into an age string.
     If show_short_version is True, then it will generate a not so accurate but shorter string,
     example: 2days ago, instead of 2 days and 23 hours ago.
 
     :param prevdate: datetime object
     :param show_short_version: if it should aproximate the date and return a shorter string
     :rtype: unicode
     :returns: unicode words describing age
     """
     now = now or datetime.datetime.now()
     order = ['year', 'month', 'day', 'hour', 'minute', 'second']
     deltas = {}
     future = False
 
     if prevdate > now:
         now, prevdate = prevdate, now
         future = True
     if future:
         prevdate = prevdate.replace(microsecond=0)
     # Get date parts deltas
     from dateutil import relativedelta
     for part in order:
         d = relativedelta.relativedelta(now, prevdate)
         deltas[part] = getattr(d, part + 's')
 
     # Fix negative offsets (there is 1 second between 10:59:59 and 11:00:00,
     # not 1 hour, -59 minutes and -59 seconds)
     for num, length in [(5, 60), (4, 60), (3, 24)]:  # seconds, minutes, hours
         part = order[num]
         carry_part = order[num - 1]
 
         if deltas[part] < 0:
             deltas[part] += length
             deltas[carry_part] -= 1
 
     # Same thing for days except that the increment depends on the (variable)
     # number of days in the month
     month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
     if deltas['day'] < 0:
         if prevdate.month == 2 and (prevdate.year % 4 == 0 and
             (prevdate.year % 100 != 0 or prevdate.year % 400 == 0)):
             deltas['day'] += 29
         else:
             deltas['day'] += month_lengths[prevdate.month - 1]
 
         deltas['month'] -= 1
 
     if deltas['month'] < 0:
         deltas['month'] += 12
         deltas['year'] -= 1
 
     # Format the result
     fmt_funcs = {
         'year': lambda d: ungettext(u'%d year', '%d years', d) % d,
         'month': lambda d: ungettext(u'%d month', '%d months', d) % d,
         'day': lambda d: ungettext(u'%d day', '%d days', d) % d,
         'hour': lambda d: ungettext(u'%d hour', '%d hours', d) % d,
         'minute': lambda d: ungettext(u'%d minute', '%d minutes', d) % d,
         'second': lambda d: ungettext(u'%d second', '%d seconds', d) % d,
     }
 
     for i, part in enumerate(order):
         value = deltas[part]
         if value == 0:
             continue
 
         if i < 5:
             sub_part = order[i + 1]
             sub_value = deltas[sub_part]
         else:
             sub_value = 0
 
         if sub_value == 0 or show_short_version:
             if future:
                 return _(u'in %s') % fmt_funcs[part](value)
             else:
                 return _(u'%s ago') % fmt_funcs[part](value)
         if future:
             return _(u'in %s and %s') % (fmt_funcs[part](value),
                 fmt_funcs[sub_part](sub_value))
         else:
             return _(u'%s and %s ago') % (fmt_funcs[part](value),
                 fmt_funcs[sub_part](sub_value))
 
     return _(u'just now')
 
 
 def uri_filter(uri):
     """
     Removes user:password from given url string
 
     :param uri:
     :rtype: unicode
     :returns: filtered list of strings
     """
     if not uri:
         return ''
 
     proto = ''
 
     for pat in ('https://', 'http://'):
         if uri.startswith(pat):
             uri = uri[len(pat):]
             proto = pat
             break
 
     # remove passwords and username
     uri = uri[uri.find('@') + 1:]
 
     # get the port
     cred_pos = uri.find(':')
     if cred_pos == -1:
         host, port = uri, None
     else:
         host, port = uri[:cred_pos], uri[cred_pos + 1:]
 
     return filter(None, [proto, host, port])
 
 
 def credentials_filter(uri):
     """
     Returns a url with removed credentials
 
     :param uri:
     """
 
     uri = uri_filter(uri)
     #check if we have port
     if len(uri) > 2 and uri[2]:
         uri[2] = ':' + uri[2]
 
     return ''.join(uri)
 
 
 def get_changeset_safe(repo, rev):
     """
     Safe version of get_changeset if this changeset doesn't exists for a
     repo it returns a Dummy one instead
 
     :param repo:
     :param rev:
     """
     from rhodecode.lib.vcs.backends.base import BaseRepository
     from rhodecode.lib.vcs.exceptions import RepositoryError
     from rhodecode.lib.vcs.backends.base import EmptyChangeset
     if not isinstance(repo, BaseRepository):
         raise Exception('You must pass an Repository '
                         'object as first argument got %s', type(repo))
 
     try:
         cs = repo.get_changeset(rev)
     except RepositoryError:
         cs = EmptyChangeset(requested_revision=rev)
     return cs
 
 
 def datetime_to_time(dt):
     if dt:
         return time.mktime(dt.timetuple())
 
 
 def time_to_datetime(tm):
     if tm:
         if isinstance(tm, basestring):
             try:
                 tm = float(tm)
             except ValueError:
                 return
         return datetime.datetime.fromtimestamp(tm)
 
 MENTIONS_REGEX = r'(?:^@|\s@)([a-zA-Z0-9]{1}[a-zA-Z0-9\-\_\.]+)(?:\s{1})'
 
 
 def extract_mentioned_users(s):
     """
     Returns unique usernames from given string s that have @mention
 
     :param s: string to get mentions
     """
     usrs = set()
     for username in re.findall(MENTIONS_REGEX, s):
         usrs.add(username)
 
     return sorted(list(usrs), key=lambda k: k.lower())
 
 
 class AttributeDict(dict):
     def __getattr__(self, attr):
         return self.get(attr, None)
     __setattr__ = dict.__setitem__
     __delattr__ = dict.__delitem__
 
 
 def fix_PATH(os_=None):
     """
     Get current active python path, and append it to PATH variable to fix issues
     of subprocess calls and different python versions
     """
     if os_ is None:
         import os
     else:
         os = os_
 
     cur_path = os.path.split(sys.executable)[0]
     if not os.environ['PATH'].startswith(cur_path):
         os.environ['PATH'] = '%s:%s' % (cur_path, os.environ['PATH'])
 
 
 def obfuscate_url_pw(engine):
     _url = engine or ''
     from sqlalchemy.engine import url as sa_url
     try:
         _url = sa_url.make_url(engine)
         if _url.password:
             _url.password = 'XXXXX'
     except Exception:
         pass
     return str(_url)
 
 
 def get_server_url(environ):
     req = webob.Request(environ)
     return req.host_url + req.script_name
 
 
 def _extract_extras(env=None):
     """
     Extracts the rc extras data from os.environ, and wraps it into named
     AttributeDict object
     """
     if not env:
         env = os.environ
 
     try:
         rc_extras = json.loads(env['RC_SCM_DATA'])
     except Exception:
         print os.environ
         print >> sys.stderr, traceback.format_exc()
         rc_extras = {}
 
     try:
         for k in ['username', 'repository', 'locked_by', 'scm', 'make_lock',
                   'action', 'ip']:
             rc_extras[k]
     except KeyError, e:
         raise Exception('Missing key %s in os.environ %s' % (e, rc_extras))
 
     return AttributeDict(rc_extras)
 
 
 def _set_extras(extras):
     os.environ['RC_SCM_DATA'] = json.dumps(extras)
+
+
+def unique_id(hexlen=32):
+    alphabet = "23456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghjklmnpqrstuvwxyz"
+    return suuid(truncate_to=hexlen, alphabet=alphabet)
+
+
+def suuid(url=None, truncate_to=22, alphabet=None):
+    """
+    Generate and return a short URL safe UUID.
+
+    If the url parameter is provided, set the namespace to the provided
+    URL and generate a UUID.
+
+    :param url to get the uuid for
+    :truncate_to: truncate the basic 22 UUID to shorter version
+
+    The IDs won't be universally unique any longer, but the probability of
+    a collision will still be very low.
+    """
+    # Define our alphabet.
+    _ALPHABET = alphabet or "23456789ABCDEFGHJKLMNPQRSTUVWXYZ"
+
+    # If no URL is given, generate a random UUID.
+    if url is None:
+        unique_id = uuid.uuid4().int
+    else:
+        unique_id = uuid.uuid3(uuid.NAMESPACE_URL, url).int
+
+    alphabet_length = len(_ALPHABET)
+    output = []
+    while unique_id > 0:
+        digit = unique_id % alphabet_length
+        output.append(_ALPHABET[digit])
+        unique_id = int(unique_id / alphabet_length)
+    return "".join(output)[:truncate_to]

rhodecode/model/__init__.py

@@ -1,140 +1,137 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.model.__init__
     ~~~~~~~~~~~~~~~~~~~~~~~~
 
     The application's model objects
 
     :created_on: Nov 25, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 
 
     :example:
 
         .. code-block:: python
 
            from paste.deploy import appconfig
            from pylons import config
            from sqlalchemy import engine_from_config
            from rhodecode.config.environment import load_environment
 
            conf = appconfig('config:development.ini', relative_to = './../../')
            load_environment(conf.global_conf, conf.local_conf)
 
            engine = engine_from_config(config, 'sqlalchemy.')
            init_model(engine)
            # RUN YOUR CODE HERE
 
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
 from rhodecode.model import meta
 from rhodecode.lib.utils2 import safe_str, obfuscate_url_pw
 
 log = logging.getLogger(__name__)
 
 
 def init_model(engine):
     """
     Initializes db session, bind the engine with the metadata,
     Call this before using any of the tables or classes in the model,
     preferably once in application start
 
     :param engine: engine to bind to
     """
     engine_str = obfuscate_url_pw(str(engine.url))
     log.info("initializing db for %s" % engine_str)
     meta.Base.metadata.bind = engine
 
 
 class BaseModel(object):
     """
     Base Model for all RhodeCode models, it adds sql alchemy session
     into instance of model
 
     :param sa: If passed it reuses this session instead of creating a new one
     """
 
     cls = None  # override in child class
 
     def __init__(self, sa=None):
         if sa is not None:
             self.sa = sa
         else:
             self.sa = meta.Session()
 
     def _get_instance(self, cls, instance, callback=None):
         """
         Get's instance of given cls using some simple lookup mechanism.
 
         :param cls: class to fetch
         :param instance: int or Instance
         :param callback: callback to call if all lookups failed
         """
 
         if isinstance(instance, cls):
             return instance
         elif isinstance(instance, (int, long)) or safe_str(instance).isdigit():
             return cls.get(instance)
         else:
             if instance:
                 if callback is None:
                     raise Exception(
                         'given object must be int, long or Instance of %s '
                         'got %s, no callback provided' % (cls, type(instance))
                     )
                 else:
                     return callback(instance)
 
     def _get_user(self, user):
         """
         Helper method to get user by ID, or username fallback
 
-        :param user:
-        :type user: UserID, username, or User instance
+        :param user: UserID, username, or User instance
         """
         from rhodecode.model.db import User
         return self._get_instance(User, user,
                                   callback=User.get_by_username)
 
     def _get_repo(self, repository):
         """
         Helper method to get repository by ID, or repository name
 
-        :param repository:
-        :type repository: RepoID, repository name or Repository Instance
+        :param repository: RepoID, repository name or Repository Instance
         """
         from rhodecode.model.db import Repository
         return self._get_instance(Repository, repository,
                                   callback=Repository.get_by_repo_name)
 
     def _get_perm(self, permission):
         """
         Helper method to get permission by ID, or permission name
 
-        :param permission:
-        :type permission: PermissionID, permission_name or Permission instance
+        :param permission: PermissionID, permission_name or Permission instance
         """
         from rhodecode.model.db import Permission
         return self._get_instance(Permission, permission,
                                   callback=Permission.get_by_key)
 
     def get_all(self):
         """
         Returns all instances of what is defined in `cls` class variable
         """
         return self.cls.getAll()

rhodecode/model/db.py

@@ -1,2133 +1,2170 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.model.db
     ~~~~~~~~~~~~~~~~~~
 
     Database Models for RhodeCode
 
     :created_on: Apr 08, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import logging
 import datetime
 import traceback
 import hashlib
 import time
 from collections import defaultdict
 
 from sqlalchemy import *
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import relationship, joinedload, class_mapper, validates
 from sqlalchemy.exc import DatabaseError
 from beaker.cache import cache_region, region_invalidate
 from webob.exc import HTTPNotFound
 
 from pylons.i18n.translation import lazy_ugettext as _
 
 from rhodecode.lib.vcs import get_backend
 from rhodecode.lib.vcs.utils.helpers import get_scm
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.utils.lazy import LazyProperty
 from rhodecode.lib.vcs.backends.base import EmptyChangeset
 
 from rhodecode.lib.utils2 import str2bool, safe_str, get_changeset_safe, \
     safe_unicode, remove_suffix, remove_prefix, time_to_datetime, _set_extras
 from rhodecode.lib.compat import json
 from rhodecode.lib.caching_query import FromCache
 
 from rhodecode.model.meta import Base, Session
 
 URL_SEP = '/'
 log = logging.getLogger(__name__)
 
 #==============================================================================
 # BASE CLASSES
 #==============================================================================
 
 _hash_key = lambda k: hashlib.md5(safe_str(k)).hexdigest()
 
 
 class BaseModel(object):
     """
     Base Model for all classess
     """
 
     @classmethod
     def _get_keys(cls):
         """return column names for this model """
         return class_mapper(cls).c.keys()
 
     def get_dict(self):
         """
         return dict with keys and values corresponding
         to this model data """
 
         d = {}
         for k in self._get_keys():
             d[k] = getattr(self, k)
 
         # also use __json__() if present to get additional fields
         _json_attr = getattr(self, '__json__', None)
         if _json_attr:
             # update with attributes from __json__
             if callable(_json_attr):
                 _json_attr = _json_attr()
             for k, val in _json_attr.iteritems():
                 d[k] = val
         return d
 
     def get_appstruct(self):
         """return list with keys and values tupples corresponding
         to this model data """
 
         l = []
         for k in self._get_keys():
             l.append((k, getattr(self, k),))
         return l
 
     def populate_obj(self, populate_dict):
         """populate model with data from given populate_dict"""
 
         for k in self._get_keys():
             if k in populate_dict:
                 setattr(self, k, populate_dict[k])
 
     @classmethod
     def query(cls):
         return Session().query(cls)
 
     @classmethod
     def get(cls, id_):
         if id_:
             return cls.query().get(id_)
 
     @classmethod
     def get_or_404(cls, id_):
         try:
             id_ = int(id_)
         except (TypeError, ValueError):
             raise HTTPNotFound
 
         res = cls.query().get(id_)
         if not res:
             raise HTTPNotFound
         return res
 
     @classmethod
     def getAll(cls):
         # deprecated and left for backward compatibility
         return cls.get_all()
 
     @classmethod
     def get_all(cls):
         return cls.query().all()
 
     @classmethod
     def delete(cls, id_):
         obj = cls.query().get(id_)
         Session().delete(obj)
 
     def __repr__(self):
         if hasattr(self, '__unicode__'):
             # python repr needs to return str
             return safe_str(self.__unicode__())
         return '<DB:%s>' % (self.__class__.__name__)
 
 
 class RhodeCodeSetting(Base, BaseModel):
     __tablename__ = 'rhodecode_settings'
     __table_args__ = (
         UniqueConstraint('app_settings_name'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'}
     )
     app_settings_id = Column("app_settings_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     app_settings_name = Column("app_settings_name", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     _app_settings_value = Column("app_settings_value", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
     def __init__(self, k='', v=''):
         self.app_settings_name = k
         self.app_settings_value = v
 
     @validates('_app_settings_value')
     def validate_settings_value(self, key, val):
         assert type(val) == unicode
         return val
 
     @hybrid_property
     def app_settings_value(self):
         v = self._app_settings_value
         if self.app_settings_name in ["ldap_active",
                                       "default_repo_enable_statistics",
                                       "default_repo_enable_locking",
                                       "default_repo_private",
                                       "default_repo_enable_downloads"]:
             v = str2bool(v)
         return v
 
     @app_settings_value.setter
     def app_settings_value(self, val):
         """
         Setter that will always make sure we use unicode in app_settings_value
 
         :param val:
         """
         self._app_settings_value = safe_unicode(val)
 
     def __unicode__(self):
         return u"<%s('%s:%s')>" % (
             self.__class__.__name__,
             self.app_settings_name, self.app_settings_value
         )
 
     @classmethod
     def get_by_name(cls, key):
         return cls.query()\
             .filter(cls.app_settings_name == key).scalar()
 
     @classmethod
     def get_by_name_or_create(cls, key):
         res = cls.get_by_name(key)
         if not res:
             res = cls(key)
         return res
 
     @classmethod
     def get_app_settings(cls, cache=False):
 
         ret = cls.query()
 
         if cache:
             ret = ret.options(FromCache("sql_cache_short", "get_hg_settings"))
 
         if not ret:
             raise Exception('Could not get application settings !')
         settings = {}
         for each in ret:
             settings['rhodecode_' + each.app_settings_name] = \
                 each.app_settings_value
 
         return settings
 
     @classmethod
     def get_ldap_settings(cls, cache=False):
         ret = cls.query()\
                 .filter(cls.app_settings_name.startswith('ldap_')).all()
         fd = {}
         for row in ret:
             fd.update({row.app_settings_name: row.app_settings_value})
 
         return fd
 
     @classmethod
     def get_default_repo_settings(cls, cache=False, strip_prefix=False):
         ret = cls.query()\
                 .filter(cls.app_settings_name.startswith('default_')).all()
         fd = {}
         for row in ret:
             key = row.app_settings_name
             if strip_prefix:
                 key = remove_prefix(key, prefix='default_')
             fd.update({key: row.app_settings_value})
 
         return fd
 
 
 class RhodeCodeUi(Base, BaseModel):
     __tablename__ = 'rhodecode_ui'
     __table_args__ = (
         UniqueConstraint('ui_key'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'}
     )
 
     HOOK_UPDATE = 'changegroup.update'
     HOOK_REPO_SIZE = 'changegroup.repo_size'
     HOOK_PUSH = 'changegroup.push_logger'
     HOOK_PRE_PUSH = 'prechangegroup.pre_push'
     HOOK_PULL = 'outgoing.pull_logger'
     HOOK_PRE_PULL = 'preoutgoing.pre_pull'
 
     ui_id = Column("ui_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     ui_section = Column("ui_section", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_key = Column("ui_key", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_value = Column("ui_value", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_active = Column("ui_active", Boolean(), nullable=True, unique=None, default=True)
 
     @classmethod
     def get_by_key(cls, key):
         return cls.query().filter(cls.ui_key == key).scalar()
 
     @classmethod
     def get_builtin_hooks(cls):
         q = cls.query()
         q = q.filter(cls.ui_key.in_([cls.HOOK_UPDATE, cls.HOOK_REPO_SIZE,
                                      cls.HOOK_PUSH, cls.HOOK_PRE_PUSH,
                                      cls.HOOK_PULL, cls.HOOK_PRE_PULL]))
         return q.all()
 
     @classmethod
     def get_custom_hooks(cls):
         q = cls.query()
         q = q.filter(~cls.ui_key.in_([cls.HOOK_UPDATE, cls.HOOK_REPO_SIZE,
                                       cls.HOOK_PUSH, cls.HOOK_PRE_PUSH,
                                       cls.HOOK_PULL, cls.HOOK_PRE_PULL]))
         q = q.filter(cls.ui_section == 'hooks')
         return q.all()
 
     @classmethod
     def get_repos_location(cls):
         return cls.get_by_key('/').ui_value
 
     @classmethod
     def create_or_update_hook(cls, key, val):
         new_ui = cls.get_by_key(key) or cls()
         new_ui.ui_section = 'hooks'
         new_ui.ui_active = True
         new_ui.ui_key = key
         new_ui.ui_value = val
 
         Session().add(new_ui)
 
     def __repr__(self):
         return '<DB:%s[%s:%s]>' % (self.__class__.__name__, self.ui_key,
                                    self.ui_value)
 
 
 class User(Base, BaseModel):
     __tablename__ = 'users'
     __table_args__ = (
         UniqueConstraint('username'), UniqueConstraint('email'),
         Index('u_username_idx', 'username'),
         Index('u_email_idx', 'email'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'}
     )
     DEFAULT_USER = 'default'
 
     user_id = Column("user_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     username = Column("username", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     password = Column("password", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     active = Column("active", Boolean(), nullable=True, unique=None, default=True)
     admin = Column("admin", Boolean(), nullable=True, unique=None, default=False)
     name = Column("firstname", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     lastname = Column("lastname", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     _email = Column("email", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     last_login = Column("last_login", DateTime(timezone=False), nullable=True, unique=None, default=None)
     ldap_dn = Column("ldap_dn", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     api_key = Column("api_key", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     inherit_default_permissions = Column("inherit_default_permissions", Boolean(), nullable=False, unique=None, default=True)
 
     user_log = relationship('UserLog')
     user_perms = relationship('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all')
 
     repositories = relationship('Repository')
     user_followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all')
     followings = relationship('UserFollowing', primaryjoin='UserFollowing.user_id==User.user_id', cascade='all')
 
     repo_to_perm = relationship('UserRepoToPerm', primaryjoin='UserRepoToPerm.user_id==User.user_id', cascade='all')
     repo_group_to_perm = relationship('UserRepoGroupToPerm', primaryjoin='UserRepoGroupToPerm.user_id==User.user_id', cascade='all')
 
     group_member = relationship('UserGroupMember', cascade='all')
 
     notifications = relationship('UserNotification', cascade='all')
     # notifications assigned to this user
     user_created_notifications = relationship('Notification', cascade='all')
     # comments created by this user
     user_comments = relationship('ChangesetComment', cascade='all')
     #extra emails for this user
     user_emails = relationship('UserEmailMap', cascade='all')
 
     @hybrid_property
     def email(self):
         return self._email
 
     @email.setter
     def email(self, val):
         self._email = val.lower() if val else None
 
     @property
     def firstname(self):
         # alias for future
         return self.name
 
     @property
     def emails(self):
         other = UserEmailMap.query().filter(UserEmailMap.user==self).all()
         return [self.email] + [x.email for x in other]
 
     @property
     def ip_addresses(self):
         ret = UserIpMap.query().filter(UserIpMap.user == self).all()
         return [x.ip_addr for x in ret]
 
     @property
     def username_and_name(self):
         return '%s (%s %s)' % (self.username, self.firstname, self.lastname)
 
     @property
     def full_name(self):
         return '%s %s' % (self.firstname, self.lastname)
 
     @property
     def full_name_or_username(self):
         return ('%s %s' % (self.firstname, self.lastname)
                 if (self.firstname and self.lastname) else self.username)
 
     @property
     def full_contact(self):
         return '%s %s <%s>' % (self.firstname, self.lastname, self.email)
 
     @property
     def short_contact(self):
         return '%s %s' % (self.firstname, self.lastname)
 
     @property
     def is_admin(self):
         return self.admin
 
     @property
     def AuthUser(self):
         """
         Returns instance of AuthUser for this user
         """
         from rhodecode.lib.auth import AuthUser
         return AuthUser(user_id=self.user_id, api_key=self.api_key,
                         username=self.username)
 
     def __unicode__(self):
         return u"<%s('id:%s:%s')>" % (self.__class__.__name__,
                                      self.user_id, self.username)
 
     @classmethod
     def get_by_username(cls, username, case_insensitive=False, cache=False):
         if case_insensitive:
             q = cls.query().filter(cls.username.ilike(username))
         else:
             q = cls.query().filter(cls.username == username)
 
         if cache:
             q = q.options(FromCache(
                             "sql_cache_short",
                             "get_user_%s" % _hash_key(username)
                           )
             )
         return q.scalar()
 
     @classmethod
     def get_by_api_key(cls, api_key, cache=False):
         q = cls.query().filter(cls.api_key == api_key)
 
         if cache:
             q = q.options(FromCache("sql_cache_short",
                                     "get_api_key_%s" % api_key))
         return q.scalar()
 
     @classmethod
     def get_by_email(cls, email, case_insensitive=False, cache=False):
         if case_insensitive:
             q = cls.query().filter(cls.email.ilike(email))
         else:
             q = cls.query().filter(cls.email == email)
 
         if cache:
             q = q.options(FromCache("sql_cache_short",
                                     "get_email_key_%s" % email))
 
         ret = q.scalar()
         if ret is None:
             q = UserEmailMap.query()
             # try fetching in alternate email map
             if case_insensitive:
                 q = q.filter(UserEmailMap.email.ilike(email))
             else:
                 q = q.filter(UserEmailMap.email == email)
             q = q.options(joinedload(UserEmailMap.user))
             if cache:
                 q = q.options(FromCache("sql_cache_short",
                                         "get_email_map_key_%s" % email))
             ret = getattr(q.scalar(), 'user', None)
 
         return ret
 
     @classmethod
     def get_from_cs_author(cls, author):
         """
         Tries to get User objects out of commit author string
 
         :param author:
         """
         from rhodecode.lib.helpers import email, author_name
         # Valid email in the attribute passed, see if they're in the system
         _email = email(author)
         if _email:
             user = cls.get_by_email(_email, case_insensitive=True)
             if user:
                 return user
         # Maybe we can match by username?
         _author = author_name(author)
         user = cls.get_by_username(_author, case_insensitive=True)
         if user:
             return user
 
     def update_lastlogin(self):
         """Update user lastlogin"""
         self.last_login = datetime.datetime.now()
         Session().add(self)
         log.debug('updated user %s lastlogin' % self.username)
 
     @classmethod
     def get_first_admin(cls):
         user = User.query().filter(User.admin == True).first()
         if user is None:
             raise Exception('Missing administrative account!')
         return user
 
     @classmethod
     def get_default_user(cls, cache=False):
         user = User.get_by_username(User.DEFAULT_USER, cache=cache)
         if user is None:
             raise Exception('Missing default account!')
         return user
 
     def get_api_data(self):
         """
         Common function for generating user related data for API
         """
         user = self
         data = dict(
             user_id=user.user_id,
             username=user.username,
             firstname=user.name,
             lastname=user.lastname,
             email=user.email,
             emails=user.emails,
             api_key=user.api_key,
             active=user.active,
             admin=user.admin,
             ldap_dn=user.ldap_dn,
             last_login=user.last_login,
             ip_addresses=user.ip_addresses
         )
         return data
 
     def __json__(self):
         data = dict(
             full_name=self.full_name,
             full_name_or_username=self.full_name_or_username,
             short_contact=self.short_contact,
             full_contact=self.full_contact
         )
         data.update(self.get_api_data())
         return data
 
 
 class UserEmailMap(Base, BaseModel):
     __tablename__ = 'user_email_map'
     __table_args__ = (
         Index('uem_email_idx', 'email'),
         UniqueConstraint('email'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'}
     )
     __mapper_args__ = {}
 
     email_id = Column("email_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
     _email = Column("email", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=False, default=None)
     user = relationship('User', lazy='joined')
 
     @validates('_email')
     def validate_email(self, key, email):
         # check if this email is not main one
         main_email = Session().query(User).filter(User.email == email).scalar()
         if main_email is not None:
             raise AttributeError('email %s is present is user table' % email)
         return email
 
     @hybrid_property
     def email(self):
         return self._email
 
     @email.setter
     def email(self, val):
         self._email = val.lower() if val else None
 
 
 class UserIpMap(Base, BaseModel):
     __tablename__ = 'user_ip_map'
     __table_args__ = (
         UniqueConstraint('user_id', 'ip_addr'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'}
     )
     __mapper_args__ = {}
 
     ip_id = Column("ip_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
     ip_addr = Column("ip_addr", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=False, default=None)
     active = Column("active", Boolean(), nullable=True, unique=None, default=True)
     user = relationship('User', lazy='joined')
 
     @classmethod
     def _get_ip_range(cls, ip_addr):
         from rhodecode.lib import ipaddr
         net = ipaddr.IPNetwork(address=ip_addr)
         return [str(net.network), str(net.broadcast)]
 
     def __json__(self):
         return dict(
           ip_addr=self.ip_addr,
           ip_range=self._get_ip_range(self.ip_addr)
         )
 
 
 class UserLog(Base, BaseModel):
     __tablename__ = 'user_logs'
     __table_args__ = (
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'},
     )
     user_log_id = Column("user_log_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
     username = Column("username", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=True)
     repository_name = Column("repository_name", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     user_ip = Column("user_ip", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     action = Column("action", UnicodeText(1200000, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     action_date = Column("action_date", DateTime(timezone=False), nullable=True, unique=None, default=None)
 
     @property
     def action_as_day(self):
         return datetime.date(*self.action_date.timetuple()[:3])
 
     user = relationship('User')
     repository = relationship('Repository', cascade='')
 
 
 class UserGroup(Base, BaseModel):
     __tablename__ = 'users_groups'
     __table_args__ = (
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'},
     )
 
     users_group_id = Column("users_group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_name = Column("users_group_name", String(255, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
     users_group_active = Column("users_group_active", Boolean(), nullable=True, unique=None, default=None)
     inherit_default_permissions = Column("users_group_inherit_default_permissions", Boolean(), nullable=False, unique=None, default=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None)
 
     members = relationship('UserGroupMember', cascade="all, delete, delete-orphan", lazy="joined")
     users_group_to_perm = relationship('UserGroupToPerm', cascade='all')
     users_group_repo_to_perm = relationship('UserGroupRepoToPerm', cascade='all')
     users_group_repo_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')
     user_user_group_to_perm = relationship('UserUserGroupToPerm ', cascade='all')
     user_group_user_group_to_perm = relationship('UserGroupUserGroupToPerm ', primaryjoin="UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id", cascade='all')
 
     user = relationship('User')
 
     def __unicode__(self):
         return u"<%s('id:%s:%s')>" % (self.__class__.__name__,
                                       self.users_group_id,
                                       self.users_group_name)
 
     @classmethod
     def get_by_group_name(cls, group_name, cache=False,
                           case_insensitive=False):
         if case_insensitive:
             q = cls.query().filter(cls.users_group_name.ilike(group_name))
         else:
             q = cls.query().filter(cls.users_group_name == group_name)
         if cache:
             q = q.options(FromCache(
                             "sql_cache_short",
                             "get_user_%s" % _hash_key(group_name)
                           )
             )
         return q.scalar()
 
     @classmethod
     def get(cls, users_group_id, cache=False):
         users_group = cls.query()
         if cache:
             users_group = users_group.options(FromCache("sql_cache_short",
                                     "get_users_group_%s" % users_group_id))
         return users_group.get(users_group_id)
 
     def get_api_data(self):
         users_group = self
 
         data = dict(
             users_group_id=users_group.users_group_id,
             group_name=users_group.users_group_name,
             active=users_group.users_group_active,
         )
 
         return data
 
 
 class UserGroupMember(Base, BaseModel):
     __tablename__ = 'users_groups_members'
     __table_args__ = (
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'},
     )
 
     users_group_member_id = Column("users_group_member_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User', lazy='joined')
     users_group = relationship('UserGroup')
 
     def __init__(self, gr_id='', u_id=''):
         self.users_group_id = gr_id
         self.user_id = u_id
 
 
 class RepositoryField(Base, BaseModel):
     __tablename__ = 'repositories_fields'
     __table_args__ = (
         UniqueConstraint('repository_id', 'field_key'),  # no-multi field
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'},
     )
     PREFIX = 'ex_'  # prefix used in form to not conflict with already existing fields
 
     repo_field_id = Column("repo_field_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
     field_key = Column("field_key", String(250, convert_unicode=False, assert_unicode=None))
     field_label = Column("field_label", String(1024, convert_unicode=False, assert_unicode=None), nullable=False)
     field_value = Column("field_value", String(10000, convert_unicode=False, assert_unicode=None), nullable=False)
     field_desc = Column("field_desc", String(1024, convert_unicode=False, assert_unicode=None), nullable=False)
     field_type = Column("field_type", String(256), nullable=False, unique=None)
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 
     repository = relationship('Repository')
 
     @property
     def field_key_prefixed(self):
         return 'ex_%s' % self.field_key
 
     @classmethod
     def un_prefix_key(cls, key):
         if key.startswith(cls.PREFIX):
             return key[len(cls.PREFIX):]
         return key
 
     @classmethod
     def get_by_key_name(cls, key, repo):
         row = cls.query()\
                 .filter(cls.repository == repo)\
                 .filter(cls.field_key == key).scalar()
         return row
 
 
 class Repository(Base, BaseModel):
     __tablename__ = 'repositories'
     __table_args__ = (
         UniqueConstraint('repo_name'),
         Index('r_repo_name_idx', 'repo_name'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'},
     )
 
     repo_id = Column("repo_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     repo_name = Column("repo_name", String(255, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
     clone_uri = Column("clone_uri", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=False, default=None)
     repo_type = Column("repo_type", String(255, convert_unicode=False, assert_unicode=None), nullable=False, unique=False, default=None)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None)
     private = Column("private", Boolean(), nullable=True, unique=None, default=None)
     enable_statistics = Column("statistics", Boolean(), nullable=True, unique=None, default=True)
     enable_downloads = Column("downloads", Boolean(), nullable=True, unique=None, default=True)
     description = Column("description", String(10000, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     created_on = Column('created_on', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
     updated_on = Column('updated_on', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
     landing_rev = Column("landing_revision", String(255, convert_unicode=False, assert_unicode=None), nullable=False, unique=False, default=None)
     enable_locking = Column("enable_locking", Boolean(), nullable=False, unique=None, default=False)
     _locked = Column("locked", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=False, default=None)
     _changeset_cache = Column("changeset_cache", LargeBinary(), nullable=True) #JSON data
 
     fork_id = Column("fork_id", Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=False, default=None)
     group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=False, default=None)
 
     user = relationship('User')
     fork = relationship('Repository', remote_side=repo_id)
     group = relationship('RepoGroup')
     repo_to_perm = relationship('UserRepoToPerm', cascade='all', order_by='UserRepoToPerm.repo_to_perm_id')
     users_group_to_perm = relationship('UserGroupRepoToPerm', cascade='all')
     stats = relationship('Statistics', cascade='all', uselist=False)
 
     followers = relationship('UserFollowing',
                              primaryjoin='UserFollowing.follows_repo_id==Repository.repo_id',
                              cascade='all')
     extra_fields = relationship('RepositoryField',
                                 cascade="all, delete, delete-orphan")
 
     logs = relationship('UserLog')
     comments = relationship('ChangesetComment', cascade="all, delete, delete-orphan")
 
     pull_requests_org = relationship('PullRequest',
                     primaryjoin='PullRequest.org_repo_id==Repository.repo_id',
                     cascade="all, delete, delete-orphan")
 
     pull_requests_other = relationship('PullRequest',
                     primaryjoin='PullRequest.other_repo_id==Repository.repo_id',
                     cascade="all, delete, delete-orphan")
 
     def __unicode__(self):
         return u"<%s('%s:%s')>" % (self.__class__.__name__, self.repo_id,
                                    self.repo_name)
 
     @hybrid_property
     def locked(self):
         # always should return [user_id, timelocked]
         if self._locked:
             _lock_info = self._locked.split(':')
             return int(_lock_info[0]), _lock_info[1]
         return [None, None]
 
     @locked.setter
     def locked(self, val):
         if val and isinstance(val, (list, tuple)):
             self._locked = ':'.join(map(str, val))
         else:
             self._locked = None
 
     @hybrid_property
     def changeset_cache(self):
         from rhodecode.lib.vcs.backends.base import EmptyChangeset
         dummy = EmptyChangeset().__json__()
         if not self._changeset_cache:
             return dummy
         try:
             return json.loads(self._changeset_cache)
         except TypeError:
             return dummy
 
     @changeset_cache.setter
     def changeset_cache(self, val):
         try:
             self._changeset_cache = json.dumps(val)
         except Exception:
             log.error(traceback.format_exc())
 
     @classmethod
     def url_sep(cls):
         return URL_SEP
 
     @classmethod
     def normalize_repo_name(cls, repo_name):
         """
         Normalizes os specific repo_name to the format internally stored inside
         dabatabase using URL_SEP
 
         :param cls:
         :param repo_name:
         """
         return cls.url_sep().join(repo_name.split(os.sep))
 
     @classmethod
     def get_by_repo_name(cls, repo_name):
         q = Session().query(cls).filter(cls.repo_name == repo_name)
         q = q.options(joinedload(Repository.fork))\
                 .options(joinedload(Repository.user))\
                 .options(joinedload(Repository.group))
         return q.scalar()
 
     @classmethod
     def get_by_full_path(cls, repo_full_path):
         repo_name = repo_full_path.split(cls.base_path(), 1)[-1]
         repo_name = cls.normalize_repo_name(repo_name)
         return cls.get_by_repo_name(repo_name.strip(URL_SEP))
 
     @classmethod
     def get_repo_forks(cls, repo_id):
         return cls.query().filter(Repository.fork_id == repo_id)
 
     @classmethod
     def base_path(cls):
         """
         Returns base path when all repos are stored
 
         :param cls:
         """
         q = Session().query(RhodeCodeUi)\
             .filter(RhodeCodeUi.ui_key == cls.url_sep())
         q = q.options(FromCache("sql_cache_short", "repository_repo_path"))
         return q.one().ui_value
 
     @property
     def forks(self):
         """
         Return forks of this repo
         """
         return Repository.get_repo_forks(self.repo_id)
 
     @property
     def parent(self):
         """
         Returns fork parent
         """
         return self.fork
 
     @property
     def just_name(self):
         return self.repo_name.split(Repository.url_sep())[-1]
 
     @property
     def groups_with_parents(self):
         groups = []
         if self.group is None:
             return groups
 
         cur_gr = self.group
         groups.insert(0, cur_gr)
         while 1:
             gr = getattr(cur_gr, 'parent_group', None)
             cur_gr = cur_gr.parent_group
             if gr is None:
                 break
             groups.insert(0, gr)
 
         return groups
 
     @property
     def groups_and_repo(self):
         return self.groups_with_parents, self.just_name, self.repo_name
 
     @LazyProperty
     def repo_path(self):
         """
         Returns base full path for that repository means where it actually
         exists on a filesystem
         """
         q = Session().query(RhodeCodeUi).filter(RhodeCodeUi.ui_key ==
                                               Repository.url_sep())
         q = q.options(FromCache("sql_cache_short", "repository_repo_path"))
         return q.one().ui_value
 
     @property
     def repo_full_path(self):
         p = [self.repo_path]
         # we need to split the name by / since this is how we store the
         # names in the database, but that eventually needs to be converted
         # into a valid system path
         p += self.repo_name.split(Repository.url_sep())
         return os.path.join(*map(safe_unicode, p))
 
     @property
     def cache_keys(self):
         """
         Returns associated cache keys for that repo
         """
         return CacheInvalidation.query()\
             .filter(CacheInvalidation.cache_args == self.repo_name)\
             .order_by(CacheInvalidation.cache_key)\
             .all()
 
     def get_new_name(self, repo_name):
         """
         returns new full repository name based on assigned group and new new
 
         :param group_name:
         """
         path_prefix = self.group.full_path_splitted if self.group else []
         return Repository.url_sep().join(path_prefix + [repo_name])
 
     @property
     def _ui(self):
         """
         Creates an db based ui object for this repository
         """
         from rhodecode.lib.utils import make_ui
         return make_ui('db', clear_session=False)
 
     @classmethod
     def is_valid(cls, repo_name):
         """
         returns True if given repo name is a valid filesystem repository
 
         :param cls:
         :param repo_name:
         """
         from rhodecode.lib.utils import is_valid_repo
 
         return is_valid_repo(repo_name, cls.base_path())
 
     def get_api_data(self):
         """
         Common function for generating repo api data
 
         """
         repo = self
         data = dict(
             repo_id=repo.repo_id,
             repo_name=repo.repo_name,
             repo_type=repo.repo_type,
             clone_uri=repo.clone_uri,
             private=repo.private,
             created_on=repo.created_on,
             description=repo.description,
             landing_rev=repo.landing_rev,
             owner=repo.user.username,
             fork_of=repo.fork.repo_name if repo.fork else None,
             enable_statistics=repo.enable_statistics,
             enable_locking=repo.enable_locking,
             enable_downloads=repo.enable_downloads,
             last_changeset=repo.changeset_cache,
             locked_by=User.get(self.locked[0]).get_api_data() \
                 if self.locked[0] else None,
             locked_date=time_to_datetime(self.locked[1]) \
                 if self.locked[1] else None
         )
         rc_config = RhodeCodeSetting.get_app_settings()
         repository_fields = str2bool(rc_config.get('rhodecode_repository_fields'))
         if repository_fields:
             for f in self.extra_fields:
                 data[f.field_key_prefixed] = f.field_value
 
         return data
 
     @classmethod
     def lock(cls, repo, user_id, lock_time=None):
         if not lock_time:
             lock_time = time.time()
         repo.locked = [user_id, lock_time]
         Session().add(repo)
         Session().commit()
 
     @classmethod
     def unlock(cls, repo):
         repo.locked = None
         Session().add(repo)
         Session().commit()
 
     @classmethod
     def getlock(cls, repo):
         return repo.locked
 
     @property
     def last_db_change(self):
         return self.updated_on
 
     def clone_url(self, **override):
         from pylons import url
         from urlparse import urlparse
         import urllib
         parsed_url = urlparse(url('home', qualified=True))
         default_clone_uri = '%(scheme)s://%(user)s%(pass)s%(netloc)s%(prefix)s%(path)s'
         decoded_path = safe_unicode(urllib.unquote(parsed_url.path))
         args = {
            'user': '',
            'pass': '',
            'scheme': parsed_url.scheme,
            'netloc': parsed_url.netloc,
            'prefix': decoded_path,
            'path': self.repo_name
         }
 
         args.update(override)
         return default_clone_uri % args
 
     #==========================================================================
     # SCM PROPERTIES
     #==========================================================================
 
     def get_changeset(self, rev=None):
         return get_changeset_safe(self.scm_instance, rev)
 
     def get_landing_changeset(self):
         """
         Returns landing changeset, or if that doesn't exist returns the tip
         """
         cs = self.get_changeset(self.landing_rev) or self.get_changeset()
         return cs
 
     def update_changeset_cache(self, cs_cache=None):
         """
         Update cache of last changeset for repository, keys should be::
 
             short_id
             raw_id
             revision
             message
             date
             author
 
         :param cs_cache:
         """
         from rhodecode.lib.vcs.backends.base import BaseChangeset
         if cs_cache is None:
             cs_cache = EmptyChangeset()
             # use no-cache version here
             scm_repo = self.scm_instance_no_cache()
             if scm_repo:
                 cs_cache = scm_repo.get_changeset()
 
         if isinstance(cs_cache, BaseChangeset):
             cs_cache = cs_cache.__json__()
 
         if (cs_cache != self.changeset_cache or not self.changeset_cache):
             _default = datetime.datetime.fromtimestamp(0)
             last_change = cs_cache.get('date') or _default
             log.debug('updated repo %s with new cs cache %s'
                       % (self.repo_name, cs_cache))
             self.updated_on = last_change
             self.changeset_cache = cs_cache
             Session().add(self)
             Session().commit()
         else:
             log.debug('Skipping repo:%s already with latest changes'
                       % self.repo_name)
 
     @property
     def tip(self):
         return self.get_changeset('tip')
 
     @property
     def author(self):
         return self.tip.author
 
     @property
     def last_change(self):
         return self.scm_instance.last_change
 
     def get_comments(self, revisions=None):
         """
         Returns comments for this repository grouped by revisions
 
         :param revisions: filter query by revisions only
         """
         cmts = ChangesetComment.query()\
             .filter(ChangesetComment.repo == self)
         if revisions:
             cmts = cmts.filter(ChangesetComment.revision.in_(revisions))
         grouped = defaultdict(list)
         for cmt in cmts.all():
             grouped[cmt.revision].append(cmt)
         return grouped
 
     def statuses(self, revisions=None):
         """
         Returns statuses for this repository
 
         :param revisions: list of revisions to get statuses for
-        :type revisions: list
         """
 
         statuses = ChangesetStatus.query()\
             .filter(ChangesetStatus.repo == self)\
             .filter(ChangesetStatus.version == 0)
         if revisions:
             statuses = statuses.filter(ChangesetStatus.revision.in_(revisions))
         grouped = {}
 
         #maybe we have open new pullrequest without a status ?
         stat = ChangesetStatus.STATUS_UNDER_REVIEW
         status_lbl = ChangesetStatus.get_status_lbl(stat)
         for pr in PullRequest.query().filter(PullRequest.org_repo == self).all():
             for rev in pr.revisions:
                 pr_id = pr.pull_request_id
                 pr_repo = pr.other_repo.repo_name
                 grouped[rev] = [stat, status_lbl, pr_id, pr_repo]
 
         for stat in statuses.all():
             pr_id = pr_repo = None
             if stat.pull_request:
                 pr_id = stat.pull_request.pull_request_id
                 pr_repo = stat.pull_request.other_repo.repo_name
             grouped[stat.revision] = [str(stat.status), stat.status_lbl,
                                       pr_id, pr_repo]
         return grouped
 
     def _repo_size(self):
         from rhodecode.lib import helpers as h
         log.debug('calculating repository size...')
         return h.format_byte_size(self.scm_instance.size)
 
     #==========================================================================
     # SCM CACHE INSTANCE
     #==========================================================================
 
     def set_invalidate(self):
         """
         Mark caches of this repo as invalid.
         """
         CacheInvalidation.set_invalidate(self.repo_name)
 
     def scm_instance_no_cache(self):
         return self.__get_instance()
 
     @property
     def scm_instance(self):
         import rhodecode
         full_cache = str2bool(rhodecode.CONFIG.get('vcs_full_cache'))
         if full_cache:
             return self.scm_instance_cached()
         return self.__get_instance()
 
     def scm_instance_cached(self, valid_cache_keys=None):
         @cache_region('long_term')
         def _c(repo_name):
             return self.__get_instance()
         rn = self.repo_name
 
         valid = CacheInvalidation.test_and_set_valid(rn, None, valid_cache_keys=valid_cache_keys)
         if not valid:
             log.debug('Cache for %s invalidated, getting new object' % (rn))
             region_invalidate(_c, None, rn)
         else:
             log.debug('Getting obj for %s from cache' % (rn))
         return _c(rn)
 
     def __get_instance(self):
         repo_full_path = self.repo_full_path
         try:
             alias = get_scm(repo_full_path)[0]
             log.debug('Creating instance of %s repository from %s'
                       % (alias, repo_full_path))
             backend = get_backend(alias)
         except VCSError:
             log.error(traceback.format_exc())
             log.error('Perhaps this repository is in db and not in '
                       'filesystem run rescan repositories with '
                       '"destroy old data " option from admin panel')
             return
 
         if alias == 'hg':
 
             repo = backend(safe_str(repo_full_path), create=False,
                            baseui=self._ui)
             # skip hidden web repository
             if repo._get_hidden():
                 return
         else:
             repo = backend(repo_full_path, create=False)
 
         return repo
 
 
 class RepoGroup(Base, BaseModel):
     __tablename__ = 'groups'
     __table_args__ = (
         UniqueConstraint('group_name', 'group_parent_id'),
         CheckConstraint('group_id != group_parent_id'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'},
     )
     __mapper_args__ = {'order_by': 'group_name'}
 
     group_id = Column("group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     group_name = Column("group_name", String(255, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
     group_parent_id = Column("group_parent_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None)
     group_description = Column("group_description", String(10000, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     enable_locking = Column("enable_locking", Boolean(), nullable=False, unique=None, default=False)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None)
 
     repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id')
     users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')
     parent_group = relationship('RepoGroup', remote_side=group_id)
     user = relationship('User')
 
     def __init__(self, group_name='', parent_group=None):
         self.group_name = group_name
         self.parent_group = parent_group
 
     def __unicode__(self):
         return u"<%s('id:%s:%s')>" % (self.__class__.__name__, self.group_id,
                                       self.group_name)
 
     @classmethod
     def groups_choices(cls, groups=None, show_empty_group=True):
         from webhelpers.html import literal as _literal
         if not groups:
             groups = cls.query().all()
 
         repo_groups = []
         if show_empty_group:
             repo_groups = [('-1', '-- %s --' % _('top level'))]
         sep = ' &raquo; '
         _name = lambda k: _literal(sep.join(k))
 
         repo_groups.extend([(x.group_id, _name(x.full_path_splitted))
                               for x in groups])
 
         repo_groups = sorted(repo_groups, key=lambda t: t[1].split(sep)[0])
         return repo_groups
 
     @classmethod
     def url_sep(cls):
         return URL_SEP
 
     @classmethod
     def get_by_group_name(cls, group_name, cache=False, case_insensitive=False):
         if case_insensitive:
             gr = cls.query()\
                 .filter(cls.group_name.ilike(group_name))
         else:
             gr = cls.query()\
                 .filter(cls.group_name == group_name)
         if cache:
             gr = gr.options(FromCache(
                             "sql_cache_short",
                             "get_group_%s" % _hash_key(group_name)
                             )
             )
         return gr.scalar()
 
     @property
     def parents(self):
         parents_recursion_limit = 5
         groups = []
         if self.parent_group is None:
             return groups
         cur_gr = self.parent_group
         groups.insert(0, cur_gr)
         cnt = 0
         while 1:
             cnt += 1
             gr = getattr(cur_gr, 'parent_group', None)
             cur_gr = cur_gr.parent_group
             if gr is None:
                 break
             if cnt == parents_recursion_limit:
                 # this will prevent accidental infinit loops
                 log.error('group nested more than %s' %
                           parents_recursion_limit)
                 break
 
             groups.insert(0, gr)
         return groups
 
     @property
     def children(self):
         return RepoGroup.query().filter(RepoGroup.parent_group == self)
 
     @property
     def name(self):
         return self.group_name.split(RepoGroup.url_sep())[-1]
 
     @property
     def full_path(self):
         return self.group_name
 
     @property
     def full_path_splitted(self):
         return self.group_name.split(RepoGroup.url_sep())
 
     @property
     def repositories(self):
         return Repository.query()\
                 .filter(Repository.group == self)\
                 .order_by(Repository.repo_name)
 
     @property
     def repositories_recursive_count(self):
         cnt = self.repositories.count()
 
         def children_count(group):
             cnt = 0
             for child in group.children:
                 cnt += child.repositories.count()
                 cnt += children_count(child)
             return cnt
 
         return cnt + children_count(self)
 
     def _recursive_objects(self, include_repos=True):
         all_ = []
 
         def _get_members(root_gr):
             if include_repos:
                 for r in root_gr.repositories:
                     all_.append(r)
             childs = root_gr.children.all()
             if childs:
                 for gr in childs:
                     all_.append(gr)
                     _get_members(gr)
 
         _get_members(self)
         return [self] + all_
 
     def recursive_groups_and_repos(self):
         """
         Recursive return all groups, with repositories in those groups
         """
         return self._recursive_objects()
 
     def recursive_groups(self):
         """
         Returns all children groups for this group including children of children
         """
         return self._recursive_objects(include_repos=False)
 
     def get_new_name(self, group_name):
         """
         returns new full group name based on parent and new name
 
         :param group_name:
         """
         path_prefix = (self.parent_group.full_path_splitted if
                        self.parent_group else [])
         return RepoGroup.url_sep().join(path_prefix + [group_name])
 
 
 class Permission(Base, BaseModel):
     __tablename__ = 'permissions'
     __table_args__ = (
         Index('p_perm_name_idx', 'permission_name'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'},
     )
     PERMS = [
         ('hg.admin', _('RhodeCode Administrator')),
 
         ('repository.none', _('Repository no access')),
         ('repository.read', _('Repository read access')),
         ('repository.write', _('Repository write access')),
         ('repository.admin', _('Repository admin access')),
 
         ('group.none', _('Repository group no access')),
         ('group.read', _('Repository group read access')),
         ('group.write', _('Repository group write access')),
         ('group.admin', _('Repository group admin access')),
 
         ('usergroup.none', _('User group no access')),
         ('usergroup.read', _('User group read access')),
         ('usergroup.write', _('User group write access')),
         ('usergroup.admin', _('User group admin access')),
 
         ('hg.repogroup.create.false', _('Repository Group creation disabled')),
         ('hg.repogroup.create.true', _('Repository Group creation enabled')),
 
         ('hg.usergroup.create.false', _('User Group creation disabled')),
         ('hg.usergroup.create.true', _('User Group creation enabled')),
 
         ('hg.create.none', _('Repository creation disabled')),
         ('hg.create.repository', _('Repository creation enabled')),
 
         ('hg.fork.none', _('Repository forking disabled')),
         ('hg.fork.repository', _('Repository forking enabled')),
 
         ('hg.register.none', _('Registration disabled')),
         ('hg.register.manual_activate', _('User Registration with manual account activation')),
         ('hg.register.auto_activate', _('User Registration with automatic account activation')),
 
         ('hg.extern_activate.manual', _('Manual activation of external account')),
         ('hg.extern_activate.auto', _('Automatic activation of external account')),
 
     ]
 
     #definition of system default permissions for DEFAULT user
     DEFAULT_USER_PERMISSIONS = [
         'repository.read',
         'group.read',
         'usergroup.read',
         'hg.create.repository',
         'hg.fork.repository',
         'hg.register.manual_activate',
         'hg.extern_activate.auto',
     ]
 
     # defines which permissions are more important higher the more important
     # Weight defines which permissions are more important.
     # The higher number the more important.
     PERM_WEIGHTS = {
         'repository.none': 0,
         'repository.read': 1,
         'repository.write': 3,
         'repository.admin': 4,
 
         'group.none': 0,
         'group.read': 1,
         'group.write': 3,
         'group.admin': 4,
 
         'usergroup.none': 0,
         'usergroup.read': 1,
         'usergroup.write': 3,
         'usergroup.admin': 4,
         'hg.repogroup.create.false': 0,
         'hg.repogroup.create.true': 1,
 
         'hg.usergroup.create.false': 0,
         'hg.usergroup.create.true': 1,
 
         'hg.fork.none': 0,
         'hg.fork.repository': 1,
         'hg.create.none': 0,
         'hg.create.repository': 1
     }
 
     permission_id = Column("permission_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     permission_name = Column("permission_name", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     permission_longname = Column("permission_longname", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
     def __unicode__(self):
         return u"<%s('%s:%s')>" % (
             self.__class__.__name__, self.permission_id, self.permission_name
         )
 
     @classmethod
     def get_by_key(cls, key):
         return cls.query().filter(cls.permission_name == key).scalar()
 
     @classmethod
     def get_default_perms(cls, default_user_id):
         q = Session().query(UserRepoToPerm, Repository, cls)\
          .join((Repository, UserRepoToPerm.repository_id == Repository.repo_id))\
          .join((cls, UserRepoToPerm.permission_id == cls.permission_id))\
          .filter(UserRepoToPerm.user_id == default_user_id)
 
         return q.all()
 
     @classmethod
     def get_default_group_perms(cls, default_user_id):
         q = Session().query(UserRepoGroupToPerm, RepoGroup, cls)\
          .join((RepoGroup, UserRepoGroupToPerm.group_id == RepoGroup.group_id))\
          .join((cls, UserRepoGroupToPerm.permission_id == cls.permission_id))\
          .filter(UserRepoGroupToPerm.user_id == default_user_id)
 
         return q.all()
 
     @classmethod
     def get_default_user_group_perms(cls, default_user_id):
         q = Session().query(UserUserGroupToPerm, UserGroup, cls)\
          .join((UserGroup, UserUserGroupToPerm.user_group_id == UserGroup.users_group_id))\
          .join((cls, UserUserGroupToPerm.permission_id == cls.permission_id))\
          .filter(UserUserGroupToPerm.user_id == default_user_id)
 
         return q.all()
 
 
 class UserRepoToPerm(Base, BaseModel):
     __tablename__ = 'repo_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'repository_id', 'permission_id'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'}
     )
     repo_to_perm_id = Column("repo_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User')
     repository = relationship('Repository')
     permission = relationship('Permission')
 
     @classmethod
     def create(cls, user, repository, permission):
         n = cls()
         n.user = user
         n.repository = repository
         n.permission = permission
         Session().add(n)
         return n
 
     def __unicode__(self):
         return u'<%s => %s >' % (self.user, self.repository)
 
 
 class UserUserGroupToPerm(Base, BaseModel):
     __tablename__ = 'user_user_group_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'user_group_id', 'permission_id'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'}
     )
     user_user_group_to_perm_id = Column("user_user_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     user_group_id = Column("user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User')
     user_group = relationship('UserGroup')
     permission = relationship('Permission')
 
     @classmethod
     def create(cls, user, user_group, permission):
         n = cls()
         n.user = user
         n.user_group = user_group
         n.permission = permission
         Session().add(n)
         return n
 
     def __unicode__(self):
         return u'<%s => %s >' % (self.user, self.user_group)
 
 
 class UserToPerm(Base, BaseModel):
     __tablename__ = 'user_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'permission_id'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'}
     )
     user_to_perm_id = Column("user_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User')
     permission = relationship('Permission', lazy='joined')
 
     def __unicode__(self):
         return u'<%s => %s >' % (self.user, self.permission)
 
 
 class UserGroupRepoToPerm(Base, BaseModel):
     __tablename__ = 'users_group_repo_to_perm'
     __table_args__ = (
         UniqueConstraint('repository_id', 'users_group_id', 'permission_id'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'}
     )
     users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
 
     users_group = relationship('UserGroup')
     permission = relationship('Permission')
     repository = relationship('Repository')
 
     @classmethod
     def create(cls, users_group, repository, permission):
         n = cls()
         n.users_group = users_group
         n.repository = repository
         n.permission = permission
         Session().add(n)
         return n
 
     def __unicode__(self):
         return u'<UserGroupRepoToPerm:%s => %s >' % (self.users_group, self.repository)
 
 
 class UserGroupUserGroupToPerm(Base, BaseModel):
     __tablename__ = 'user_group_user_group_to_perm'
     __table_args__ = (
         UniqueConstraint('target_user_group_id', 'user_group_id', 'permission_id'),
         CheckConstraint('target_user_group_id != user_group_id'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'}
     )
     user_group_user_group_to_perm_id = Column("user_group_user_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     target_user_group_id = Column("target_user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     user_group_id = Column("user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
 
     target_user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id')
     user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.user_group_id==UserGroup.users_group_id')
     permission = relationship('Permission')
 
     @classmethod
     def create(cls, target_user_group, user_group, permission):
         n = cls()
         n.target_user_group = target_user_group
         n.user_group = user_group
         n.permission = permission
         Session().add(n)
         return n
 
     def __unicode__(self):
         return u'<UserGroupUserGroup:%s => %s >' % (self.target_user_group, self.user_group)
 
 
 class UserGroupToPerm(Base, BaseModel):
     __tablename__ = 'users_group_to_perm'
     __table_args__ = (
         UniqueConstraint('users_group_id', 'permission_id',),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'}
     )
     users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
     users_group = relationship('UserGroup')
     permission = relationship('Permission')
 
 
 class UserRepoGroupToPerm(Base, BaseModel):
     __tablename__ = 'user_repo_group_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'group_id', 'permission_id'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'}
     )
 
     group_to_perm_id = Column("group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User')
     group = relationship('RepoGroup')
     permission = relationship('Permission')
 
 
 class UserGroupRepoGroupToPerm(Base, BaseModel):
     __tablename__ = 'users_group_repo_group_to_perm'
     __table_args__ = (
         UniqueConstraint('users_group_id', 'group_id'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'}
     )
 
     users_group_repo_group_to_perm_id = Column("users_group_repo_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
     users_group = relationship('UserGroup')
     permission = relationship('Permission')
     group = relationship('RepoGroup')
 
 
 class Statistics(Base, BaseModel):
     __tablename__ = 'statistics'
     __table_args__ = (
          UniqueConstraint('repository_id'),
          {'extend_existing': True, 'mysql_engine': 'InnoDB',
           'mysql_charset': 'utf8'}
     )
     stat_id = Column("stat_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=True, default=None)
     stat_on_revision = Column("stat_on_revision", Integer(), nullable=False)
     commit_activity = Column("commit_activity", LargeBinary(1000000), nullable=False)#JSON data
     commit_activity_combined = Column("commit_activity_combined", LargeBinary(), nullable=False)#JSON data
     languages = Column("languages", LargeBinary(1000000), nullable=False)#JSON data
 
     repository = relationship('Repository', single_parent=True)
 
 
 class UserFollowing(Base, BaseModel):
     __tablename__ = 'user_followings'
     __table_args__ = (
         UniqueConstraint('user_id', 'follows_repository_id'),
         UniqueConstraint('user_id', 'follows_user_id'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'}
     )
 
     user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=None, default=None)
     follows_user_id = Column("follows_user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
     follows_from = Column('follows_from', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
 
     user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id')
 
     follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
     follows_repository = relationship('Repository', order_by='Repository.repo_name')
 
     @classmethod
     def get_repo_followers(cls, repo_id):
         return cls.query().filter(cls.follows_repo_id == repo_id)
 
 
 class CacheInvalidation(Base, BaseModel):
     __tablename__ = 'cache_invalidation'
     __table_args__ = (
         UniqueConstraint('cache_key'),
         Index('key_idx', 'cache_key'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'},
     )
     # cache_id, not used
     cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     # cache_key as created by _get_cache_key
     cache_key = Column("cache_key", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     # cache_args is a repo_name
     cache_args = Column("cache_args", String(255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     # instance sets cache_active True when it is caching,
     # other instances set cache_active to False to indicate that this cache is invalid
     cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)
 
     def __init__(self, cache_key, repo_name=''):
         self.cache_key = cache_key
         self.cache_args = repo_name
         self.cache_active = False
 
     def __unicode__(self):
         return u"<%s('%s:%s[%s]')>" % (self.__class__.__name__,
                             self.cache_id, self.cache_key, self.cache_active)
 
     def _cache_key_partition(self):
         prefix, repo_name, suffix = self.cache_key.partition(self.cache_args)
         return prefix, repo_name, suffix
 
     def get_prefix(self):
         """
         get prefix that might have been used in _get_cache_key to
         generate self.cache_key. Only used for informational purposes
         in repo_edit.html.
         """
         # prefix, repo_name, suffix
         return self._cache_key_partition()[0]
 
     def get_suffix(self):
         """
         get suffix that might have been used in _get_cache_key to
         generate self.cache_key. Only used for informational purposes
         in repo_edit.html.
         """
         # prefix, repo_name, suffix
         return self._cache_key_partition()[2]
 
     @classmethod
     def clear_cache(cls):
         """
         Delete all cache keys from database.
         Should only be run when all instances are down and all entries thus stale.
         """
         cls.query().delete()
         Session().commit()
 
     @classmethod
     def _get_cache_key(cls, key):
         """
         Wrapper for generating a unique cache key for this instance and "key".
         key must / will start with a repo_name which will be stored in .cache_args .
         """
         import rhodecode
         prefix = rhodecode.CONFIG.get('instance_id', '')
         return "%s%s" % (prefix, key)
 
     @classmethod
     def set_invalidate(cls, repo_name):
         """
         Mark all caches of a repo as invalid in the database.
         """
         inv_objs = Session().query(cls).filter(cls.cache_args == repo_name).all()
 
         try:
             for inv_obj in inv_objs:
                 log.debug('marking %s key for invalidation based on repo_name=%s'
                           % (inv_obj, safe_str(repo_name)))
                 inv_obj.cache_active = False
                 Session().add(inv_obj)
             Session().commit()
         except Exception:
             log.error(traceback.format_exc())
             Session().rollback()
 
     @classmethod
     def test_and_set_valid(cls, repo_name, kind, valid_cache_keys=None):
         """
         Mark this cache key as active and currently cached.
         Return True if the existing cache registration still was valid.
         Return False to indicate that it had been invalidated and caches should be refreshed.
         """
 
         key = (repo_name + '_' + kind) if kind else repo_name
         cache_key = cls._get_cache_key(key)
 
         if valid_cache_keys and cache_key in valid_cache_keys:
             return True
 
         try:
             inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar()
             if not inv_obj:
                 inv_obj = CacheInvalidation(cache_key, repo_name)
             was_valid = inv_obj.cache_active
             inv_obj.cache_active = True
             Session().add(inv_obj)
             Session().commit()
             return was_valid
         except Exception:
             log.error(traceback.format_exc())
             Session().rollback()
             return False
 
     @classmethod
     def get_valid_cache_keys(cls):
         """
         Return opaque object with information of which caches still are valid
         and can be used without checking for invalidation.
         """
         return set(inv_obj.cache_key for inv_obj in cls.query().filter(cls.cache_active).all())
 
 
 class ChangesetComment(Base, BaseModel):
     __tablename__ = 'changeset_comments'
     __table_args__ = (
         Index('cc_revision_idx', 'revision'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'},
     )
     comment_id = Column('comment_id', Integer(), nullable=False, primary_key=True)
     repo_id = Column('repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     revision = Column('revision', String(40), nullable=True)
     pull_request_id = Column("pull_request_id", Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True)
     line_no = Column('line_no', Unicode(10), nullable=True)
     hl_lines = Column('hl_lines', Unicode(512), nullable=True)
     f_path = Column('f_path', Unicode(1000), nullable=True)
     user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     text = Column('text', UnicodeText(25000), nullable=False)
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     modified_at = Column('modified_at', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 
     author = relationship('User', lazy='joined')
     repo = relationship('Repository')
     status_change = relationship('ChangesetStatus', cascade="all, delete, delete-orphan")
     pull_request = relationship('PullRequest', lazy='joined')
 
     @classmethod
     def get_users(cls, revision=None, pull_request_id=None):
         """
         Returns user associated with this ChangesetComment. ie those
         who actually commented
 
         :param cls:
         :param revision:
         """
         q = Session().query(User)\
                 .join(ChangesetComment.author)
         if revision:
             q = q.filter(cls.revision == revision)
         elif pull_request_id:
             q = q.filter(cls.pull_request_id == pull_request_id)
         return q.all()
 
 
 class ChangesetStatus(Base, BaseModel):
     __tablename__ = 'changeset_statuses'
     __table_args__ = (
         Index('cs_revision_idx', 'revision'),
         Index('cs_version_idx', 'version'),
         UniqueConstraint('repo_id', 'revision', 'version'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'}
     )
     STATUS_NOT_REVIEWED = DEFAULT = 'not_reviewed'
     STATUS_APPROVED = 'approved'
     STATUS_REJECTED = 'rejected'
     STATUS_UNDER_REVIEW = 'under_review'
 
     STATUSES = [
         (STATUS_NOT_REVIEWED, _("Not Reviewed")),  # (no icon) and default
         (STATUS_APPROVED, _("Approved")),
         (STATUS_REJECTED, _("Rejected")),
         (STATUS_UNDER_REVIEW, _("Under Review")),
     ]
 
     changeset_status_id = Column('changeset_status_id', Integer(), nullable=False, primary_key=True)
     repo_id = Column('repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None)
     revision = Column('revision', String(40), nullable=False)
     status = Column('status', String(128), nullable=False, default=DEFAULT)
     changeset_comment_id = Column('changeset_comment_id', Integer(), ForeignKey('changeset_comments.comment_id'))
     modified_at = Column('modified_at', DateTime(), nullable=False, default=datetime.datetime.now)
     version = Column('version', Integer(), nullable=False, default=0)
     pull_request_id = Column("pull_request_id", Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True)
 
     author = relationship('User', lazy='joined')
     repo = relationship('Repository')
     comment = relationship('ChangesetComment', lazy='joined')
     pull_request = relationship('PullRequest', lazy='joined')
 
     def __unicode__(self):
         return u"<%s('%s:%s')>" % (
             self.__class__.__name__,
             self.status, self.author
         )
 
     @classmethod
     def get_status_lbl(cls, value):
         return dict(cls.STATUSES).get(value)
 
     @property
     def status_lbl(self):
         return ChangesetStatus.get_status_lbl(self.status)
 
 
 class PullRequest(Base, BaseModel):
     __tablename__ = 'pull_requests'
     __table_args__ = (
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'},
     )
 
     STATUS_NEW = u'new'
     STATUS_OPEN = u'open'
     STATUS_CLOSED = u'closed'
 
     pull_request_id = Column('pull_request_id', Integer(), nullable=False, primary_key=True)
     title = Column('title', Unicode(256), nullable=True)
     description = Column('description', UnicodeText(10240), nullable=True)
     status = Column('status', Unicode(256), nullable=False, default=STATUS_NEW)
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     updated_on = Column('updated_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None)
     _revisions = Column('revisions', UnicodeText(20500))  # 500 revisions max
     org_repo_id = Column('org_repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     org_ref = Column('org_ref', Unicode(256), nullable=False)
     other_repo_id = Column('other_repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     other_ref = Column('other_ref', Unicode(256), nullable=False)
 
     @hybrid_property
     def revisions(self):
         return self._revisions.split(':')
 
     @revisions.setter
     def revisions(self, val):
         self._revisions = ':'.join(val)
 
     @property
     def org_ref_parts(self):
         return self.org_ref.split(':')
 
     @property
     def other_ref_parts(self):
         return self.other_ref.split(':')
 
     author = relationship('User', lazy='joined')
     reviewers = relationship('PullRequestReviewers',
                              cascade="all, delete, delete-orphan")
     org_repo = relationship('Repository', primaryjoin='PullRequest.org_repo_id==Repository.repo_id')
     other_repo = relationship('Repository', primaryjoin='PullRequest.other_repo_id==Repository.repo_id')
     statuses = relationship('ChangesetStatus')
     comments = relationship('ChangesetComment',
                              cascade="all, delete, delete-orphan")
 
     def is_closed(self):
         return self.status == self.STATUS_CLOSED
 
     @property
     def last_review_status(self):
         return self.statuses[-1].status if self.statuses else ''
 
     def __json__(self):
         return dict(
           revisions=self.revisions
         )
 
 
 class PullRequestReviewers(Base, BaseModel):
     __tablename__ = 'pull_request_reviewers'
     __table_args__ = (
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'},
     )
 
     def __init__(self, user=None, pull_request=None):
         self.user = user
         self.pull_request = pull_request
 
     pull_requests_reviewers_id = Column('pull_requests_reviewers_id', Integer(), nullable=False, primary_key=True)
     pull_request_id = Column("pull_request_id", Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=False)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True)
 
     user = relationship('User')
     pull_request = relationship('PullRequest')
 
 
 class Notification(Base, BaseModel):
     __tablename__ = 'notifications'
     __table_args__ = (
         Index('notification_type_idx', 'type'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'},
     )
 
     TYPE_CHANGESET_COMMENT = u'cs_comment'
     TYPE_MESSAGE = u'message'
     TYPE_MENTION = u'mention'
     TYPE_REGISTRATION = u'registration'
     TYPE_PULL_REQUEST = u'pull_request'
     TYPE_PULL_REQUEST_COMMENT = u'pull_request_comment'
 
     notification_id = Column('notification_id', Integer(), nullable=False, primary_key=True)
     subject = Column('subject', Unicode(512), nullable=True)
     body = Column('body', UnicodeText(50000), nullable=True)
     created_by = Column("created_by", Integer(), ForeignKey('users.user_id'), nullable=True)
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     type_ = Column('type', Unicode(256))
 
     created_by_user = relationship('User')
     notifications_to_users = relationship('UserNotification', lazy='joined',
                                           cascade="all, delete, delete-orphan")
 
     @property
     def recipients(self):
         return [x.user for x in UserNotification.query()\
                 .filter(UserNotification.notification == self)\
                 .order_by(UserNotification.user_id.asc()).all()]
 
     @classmethod
     def create(cls, created_by, subject, body, recipients, type_=None):
         if type_ is None:
             type_ = Notification.TYPE_MESSAGE
 
         notification = cls()
         notification.created_by_user = created_by
         notification.subject = subject
         notification.body = body
         notification.type_ = type_
         notification.created_on = datetime.datetime.now()
 
         for u in recipients:
             assoc = UserNotification()
             assoc.notification = notification
             u.notifications.append(assoc)
         Session().add(notification)
         return notification
 
     @property
     def description(self):
         from rhodecode.model.notification import NotificationModel
         return NotificationModel().make_description(self)
 
 
 class UserNotification(Base, BaseModel):
     __tablename__ = 'user_to_notification'
     __table_args__ = (
         UniqueConstraint('user_id', 'notification_id'),
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'}
     )
     user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), primary_key=True)
     notification_id = Column("notification_id", Integer(), ForeignKey('notifications.notification_id'), primary_key=True)
     read = Column('read', Boolean, default=False)
     sent_on = Column('sent_on', DateTime(timezone=False), nullable=True, unique=None)
 
     user = relationship('User', lazy="joined")
     notification = relationship('Notification', lazy="joined",
                                 order_by=lambda: Notification.created_on.desc(),)
 
     def mark_as_read(self):
         self.read = True
         Session().add(self)
 
 
+class Gist(Base, BaseModel):
+    __tablename__ = 'gists'
+    __table_args__ = (
+        Index('g_gist_access_id_idx', 'gist_access_id'),
+        Index('g_created_on_idx', 'created_on'),
+        {'extend_existing': True, 'mysql_engine': 'InnoDB',
+         'mysql_charset': 'utf8'}
+    )
+    GIST_PUBLIC = u'public'
+    GIST_PRIVATE = u'private'
+
+    gist_id = Column('gist_id', Integer(), primary_key=True)
+    gist_access_id = Column('gist_access_id', UnicodeText(1024))
+    gist_description = Column('gist_description', UnicodeText(1024))
+    gist_owner = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=True)
+    gist_expires = Column('gist_expires', Float(), nullable=False)
+    gist_type = Column('gist_type', Unicode(128), nullable=False)
+    created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
+    modified_at = Column('modified_at', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
+
+    owner = relationship('User')
+
+    @classmethod
+    def get_or_404(cls, id_):
+        res = cls.query().filter(cls.gist_access_id == id_).scalar()
+        if not res:
+            raise HTTPNotFound
+        return res
+
+    @classmethod
+    def get_by_access_id(cls, gist_access_id):
+        return cls.query().filter(cls.gist_access_id==gist_access_id).scalar()
+
+    def gist_url(self):
+        from pylons import url
+        return url('gist', id=self.gist_access_id, qualified=True)
+
+
 class DbMigrateVersion(Base, BaseModel):
     __tablename__ = 'db_migrate_version'
     __table_args__ = (
         {'extend_existing': True, 'mysql_engine': 'InnoDB',
          'mysql_charset': 'utf8'},
     )
     repository_id = Column('repository_id', String(250), primary_key=True)
     repository_path = Column('repository_path', Text)
     version = Column('version', Integer)

rhodecode/model/forms.py

@@ -1,421 +1,434 b''
 """ this is forms validation classes
 http://formencode.org/module-formencode.validators.html
 for list off all availible validators
 
 we can create our own validators
 
 The table below outlines the options which can be used in a schema in addition to the validators themselves
 pre_validators          []     These validators will be applied before the schema
 chained_validators      []     These validators will be applied after the schema
 allow_extra_fields      False     If True, then it is not an error when keys that aren't associated with a validator are present
 filter_extra_fields     False     If True, then keys that aren't associated with a validator are removed
 if_key_missing          NoDefault If this is given, then any keys that aren't available but are expected will be replaced with this value (and then validated). This does not override a present .if_missing attribute on validators. NoDefault is a special FormEncode class to mean that no default values has been specified and therefore missing keys shouldn't take a default value.
 ignore_key_missing      False     If True, then missing keys will be missing in the result, if the validator doesn't have .if_missing on it already
 
 
 <name> = formencode.validators.<name of validator>
 <name> must equal form name
 list=[1,2,3,4,5]
 for SELECT use formencode.All(OneOf(list), Int())
 
 """
 import logging
 
 import formencode
 from formencode import All
 
 from pylons.i18n.translation import _
 
 from rhodecode.model import validators as v
 from rhodecode import BACKENDS
 
 log = logging.getLogger(__name__)
 
 
 class LoginForm(formencode.Schema):
     allow_extra_fields = True
     filter_extra_fields = True
     username = v.UnicodeString(
         strip=True,
         min=1,
         not_empty=True,
         messages={
            'empty': _(u'Please enter a login'),
            'tooShort': _(u'Enter a value %(min)i characters long or more')}
     )
 
     password = v.UnicodeString(
         strip=False,
         min=3,
         not_empty=True,
         messages={
             'empty': _(u'Please enter a password'),
             'tooShort': _(u'Enter %(min)i characters or more')}
     )
 
     remember = v.StringBoolean(if_missing=False)
 
     chained_validators = [v.ValidAuth()]
 
 
 def UserForm(edit=False, old_data={}):
     class _UserForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         username = All(v.UnicodeString(strip=True, min=1, not_empty=True),
                        v.ValidUsername(edit, old_data))
         if edit:
             new_password = All(
                 v.ValidPassword(),
                 v.UnicodeString(strip=False, min=6, not_empty=False)
             )
             password_confirmation = All(
                 v.ValidPassword(),
                 v.UnicodeString(strip=False, min=6, not_empty=False),
             )
             admin = v.StringBoolean(if_missing=False)
         else:
             password = All(
                 v.ValidPassword(),
                 v.UnicodeString(strip=False, min=6, not_empty=True)
             )
             password_confirmation = All(
                 v.ValidPassword(),
                 v.UnicodeString(strip=False, min=6, not_empty=False)
             )
 
         active = v.StringBoolean(if_missing=False)
         firstname = v.UnicodeString(strip=True, min=1, not_empty=False)
         lastname = v.UnicodeString(strip=True, min=1, not_empty=False)
         email = All(v.Email(not_empty=True), v.UniqSystemEmail(old_data))
 
         chained_validators = [v.ValidPasswordsMatch()]
 
     return _UserForm
 
 
 def UserGroupForm(edit=False, old_data={}, available_members=[]):
     class _UserGroupForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
 
         users_group_name = All(
             v.UnicodeString(strip=True, min=1, not_empty=True),
             v.ValidUserGroup(edit, old_data)
         )
 
         users_group_active = v.StringBoolean(if_missing=False)
 
         if edit:
             users_group_members = v.OneOf(
                 available_members, hideList=False, testValueList=True,
                 if_missing=None, not_empty=False
             )
 
     return _UserGroupForm
 
 
 def ReposGroupForm(edit=False, old_data={}, available_groups=[],
                    can_create_in_root=False):
     class _ReposGroupForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
 
         group_name = All(v.UnicodeString(strip=True, min=1, not_empty=True),
                                v.SlugifyName())
         group_description = v.UnicodeString(strip=True, min=1,
                                                 not_empty=False)
         if edit:
             #FIXME: do a special check that we cannot move a group to one of
             #it's children
             pass
         group_parent_id = All(v.CanCreateGroup(can_create_in_root),
                               v.OneOf(available_groups, hideList=False,
                                       testValueList=True,
                                       if_missing=None, not_empty=True))
         enable_locking = v.StringBoolean(if_missing=False)
         chained_validators = [v.ValidReposGroup(edit, old_data)]
 
     return _ReposGroupForm
 
 
 def RegisterForm(edit=False, old_data={}):
     class _RegisterForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         username = All(
             v.ValidUsername(edit, old_data),
             v.UnicodeString(strip=True, min=1, not_empty=True)
         )
         password = All(
             v.ValidPassword(),
             v.UnicodeString(strip=False, min=6, not_empty=True)
         )
         password_confirmation = All(
             v.ValidPassword(),
             v.UnicodeString(strip=False, min=6, not_empty=True)
         )
         active = v.StringBoolean(if_missing=False)
         firstname = v.UnicodeString(strip=True, min=1, not_empty=False)
         lastname = v.UnicodeString(strip=True, min=1, not_empty=False)
         email = All(v.Email(not_empty=True), v.UniqSystemEmail(old_data))
 
         chained_validators = [v.ValidPasswordsMatch()]
 
     return _RegisterForm
 
 
 def PasswordResetForm():
     class _PasswordResetForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         email = All(v.ValidSystemEmail(), v.Email(not_empty=True))
     return _PasswordResetForm
 
 
 def RepoForm(edit=False, old_data={}, supported_backends=BACKENDS.keys(),
              repo_groups=[], landing_revs=[]):
     class _RepoForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         repo_name = All(v.UnicodeString(strip=True, min=1, not_empty=True),
                         v.SlugifyName())
         repo_group = All(v.CanWriteGroup(old_data),
                          v.OneOf(repo_groups, hideList=True))
         repo_type = v.OneOf(supported_backends)
         repo_description = v.UnicodeString(strip=True, min=1, not_empty=False)
         repo_private = v.StringBoolean(if_missing=False)
         repo_landing_rev = v.OneOf(landing_revs, hideList=True)
         clone_uri = All(v.UnicodeString(strip=True, min=1, not_empty=False))
 
         repo_enable_statistics = v.StringBoolean(if_missing=False)
         repo_enable_downloads = v.StringBoolean(if_missing=False)
         repo_enable_locking = v.StringBoolean(if_missing=False)
 
         if edit:
             #this is repo owner
             user = All(v.UnicodeString(not_empty=True), v.ValidRepoUser())
 
         chained_validators = [v.ValidCloneUri(),
                               v.ValidRepoName(edit, old_data)]
     return _RepoForm
 
 
 def RepoPermsForm():
     class _RepoPermsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         chained_validators = [v.ValidPerms(type_='repo')]
     return _RepoPermsForm
 
 
 def RepoGroupPermsForm():
     class _RepoGroupPermsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         recursive = v.StringBoolean(if_missing=False)
         chained_validators = [v.ValidPerms(type_='repo_group')]
     return _RepoGroupPermsForm
 
 
 def UserGroupPermsForm():
     class _UserPermsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         chained_validators = [v.ValidPerms(type_='user_group')]
     return _UserPermsForm
 
 
 def RepoFieldForm():
     class _RepoFieldForm(formencode.Schema):
         filter_extra_fields = True
         allow_extra_fields = True
 
         new_field_key = All(v.FieldKey(),
                             v.UnicodeString(strip=True, min=3, not_empty=True))
         new_field_value = v.UnicodeString(not_empty=False, if_missing='')
         new_field_type = v.OneOf(['str', 'unicode', 'list', 'tuple'],
                                  if_missing='str')
         new_field_label = v.UnicodeString(not_empty=False)
         new_field_desc = v.UnicodeString(not_empty=False)
 
     return _RepoFieldForm
 
 
 def RepoForkForm(edit=False, old_data={}, supported_backends=BACKENDS.keys(),
                  repo_groups=[], landing_revs=[]):
     class _RepoForkForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         repo_name = All(v.UnicodeString(strip=True, min=1, not_empty=True),
                         v.SlugifyName())
         repo_group = All(v.CanWriteGroup(),
                          v.OneOf(repo_groups, hideList=True))
         repo_type = All(v.ValidForkType(old_data), v.OneOf(supported_backends))
         description = v.UnicodeString(strip=True, min=1, not_empty=True)
         private = v.StringBoolean(if_missing=False)
         copy_permissions = v.StringBoolean(if_missing=False)
         update_after_clone = v.StringBoolean(if_missing=False)
         fork_parent_id = v.UnicodeString()
         chained_validators = [v.ValidForkName(edit, old_data)]
         landing_rev = v.OneOf(landing_revs, hideList=True)
 
     return _RepoForkForm
 
 
 def ApplicationSettingsForm():
     class _ApplicationSettingsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         rhodecode_title = v.UnicodeString(strip=True, min=1, not_empty=True)
         rhodecode_realm = v.UnicodeString(strip=True, min=1, not_empty=True)
         rhodecode_ga_code = v.UnicodeString(strip=True, min=1, not_empty=False)
 
     return _ApplicationSettingsForm
 
 
 def ApplicationVisualisationForm():
     class _ApplicationVisualisationForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         rhodecode_show_public_icon = v.StringBoolean(if_missing=False)
         rhodecode_show_private_icon = v.StringBoolean(if_missing=False)
         rhodecode_stylify_metatags = v.StringBoolean(if_missing=False)
 
         rhodecode_repository_fields = v.StringBoolean(if_missing=False)
         rhodecode_lightweight_journal = v.StringBoolean(if_missing=False)
 
     return _ApplicationVisualisationForm
 
 
 def ApplicationUiSettingsForm():
     class _ApplicationUiSettingsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         web_push_ssl = v.StringBoolean(if_missing=False)
         paths_root_path = All(
             v.ValidPath(),
             v.UnicodeString(strip=True, min=1, not_empty=True)
         )
         hooks_changegroup_update = v.StringBoolean(if_missing=False)
         hooks_changegroup_repo_size = v.StringBoolean(if_missing=False)
         hooks_changegroup_push_logger = v.StringBoolean(if_missing=False)
         hooks_outgoing_pull_logger = v.StringBoolean(if_missing=False)
 
         extensions_largefiles = v.StringBoolean(if_missing=False)
         extensions_hgsubversion = v.StringBoolean(if_missing=False)
         extensions_hggit = v.StringBoolean(if_missing=False)
 
     return _ApplicationUiSettingsForm
 
 
 def DefaultPermissionsForm(repo_perms_choices, group_perms_choices,
                            user_group_perms_choices, create_choices,
                            repo_group_create_choices, user_group_create_choices,
                            fork_choices, register_choices, extern_activate_choices):
     class _DefaultPermissionsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         overwrite_default_repo = v.StringBoolean(if_missing=False)
         overwrite_default_group = v.StringBoolean(if_missing=False)
         overwrite_default_user_group = v.StringBoolean(if_missing=False)
         anonymous = v.StringBoolean(if_missing=False)
         default_repo_perm = v.OneOf(repo_perms_choices)
         default_group_perm = v.OneOf(group_perms_choices)
         default_user_group_perm = v.OneOf(user_group_perms_choices)
 
         default_repo_create = v.OneOf(create_choices)
         default_user_group_create = v.OneOf(user_group_create_choices)
         #default_repo_group_create = v.OneOf(repo_group_create_choices) #not impl. yet
         default_fork = v.OneOf(fork_choices)
 
         default_register = v.OneOf(register_choices)
         default_extern_activate = v.OneOf(extern_activate_choices)
     return _DefaultPermissionsForm
 
 
 def CustomDefaultPermissionsForm():
     class _CustomDefaultPermissionsForm(formencode.Schema):
         filter_extra_fields = True
         allow_extra_fields = True
         inherit_default_permissions = v.StringBoolean(if_missing=False)
 
         create_repo_perm = v.StringBoolean(if_missing=False)
         create_user_group_perm = v.StringBoolean(if_missing=False)
         #create_repo_group_perm Impl. later
 
         fork_repo_perm = v.StringBoolean(if_missing=False)
 
     return _CustomDefaultPermissionsForm
 
 
 def DefaultsForm(edit=False, old_data={}, supported_backends=BACKENDS.keys()):
     class _DefaultsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         default_repo_type = v.OneOf(supported_backends)
         default_repo_private = v.StringBoolean(if_missing=False)
         default_repo_enable_statistics = v.StringBoolean(if_missing=False)
         default_repo_enable_downloads = v.StringBoolean(if_missing=False)
         default_repo_enable_locking = v.StringBoolean(if_missing=False)
 
     return _DefaultsForm
 
 
 def LdapSettingsForm(tls_reqcert_choices, search_scope_choices,
                      tls_kind_choices):
     class _LdapSettingsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         #pre_validators = [LdapLibValidator]
         ldap_active = v.StringBoolean(if_missing=False)
         ldap_host = v.UnicodeString(strip=True,)
         ldap_port = v.Number(strip=True,)
         ldap_tls_kind = v.OneOf(tls_kind_choices)
         ldap_tls_reqcert = v.OneOf(tls_reqcert_choices)
         ldap_dn_user = v.UnicodeString(strip=True,)
         ldap_dn_pass = v.UnicodeString(strip=True,)
         ldap_base_dn = v.UnicodeString(strip=True,)
         ldap_filter = v.UnicodeString(strip=True,)
         ldap_search_scope = v.OneOf(search_scope_choices)
         ldap_attr_login = v.AttrLoginValidator()(not_empty=True)
         ldap_attr_firstname = v.UnicodeString(strip=True,)
         ldap_attr_lastname = v.UnicodeString(strip=True,)
         ldap_attr_email = v.UnicodeString(strip=True,)
 
     return _LdapSettingsForm
 
 
 def UserExtraEmailForm():
     class _UserExtraEmailForm(formencode.Schema):
         email = All(v.UniqSystemEmail(), v.Email(not_empty=True))
     return _UserExtraEmailForm
 
 
 def UserExtraIpForm():
     class _UserExtraIpForm(formencode.Schema):
         ip = v.ValidIp()(not_empty=True)
     return _UserExtraIpForm
 
 
 def PullRequestForm(repo_id):
     class _PullRequestForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
 
         user = v.UnicodeString(strip=True, required=True)
         org_repo = v.UnicodeString(strip=True, required=True)
         org_ref = v.UnicodeString(strip=True, required=True)
         other_repo = v.UnicodeString(strip=True, required=True)
         other_ref = v.UnicodeString(strip=True, required=True)
         revisions = All(#v.NotReviewedRevisions(repo_id)(),
                         v.UniqueList(not_empty=True))
         review_members = v.UniqueList(not_empty=True)
 
         pullrequest_title = v.UnicodeString(strip=True, required=True, min=3)
         pullrequest_desc = v.UnicodeString(strip=True, required=False)
 
         ancestor_rev = v.UnicodeString(strip=True, required=True)
         merge_rev = v.UnicodeString(strip=True, required=True)
 
     return _PullRequestForm
+
+
+def GistForm(lifetime_options):
+    class _GistForm(formencode.Schema):
+
+        filename = v.UnicodeString(strip=True, required=False)
+        description = v.UnicodeString(required=False, if_missing='')
+        lifetime = v.OneOf(lifetime_options)
+        content = v.UnicodeString(required=True, not_empty=True)
+        public = v.UnicodeString(required=False, if_missing='')
+        private = v.UnicodeString(required=False, if_missing='')
+
+    return _GistForm

rhodecode/model/repo.py

@@ -1,746 +1,755 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.model.repo
     ~~~~~~~~~~~~~~~~~~~~
 
     Repository model for rhodecode
 
     :created_on: Jun 5, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import with_statement
 import os
 import shutil
 import logging
 import traceback
 from datetime import datetime
 
 from rhodecode.lib.vcs.backends import get_backend
 from rhodecode.lib.compat import json
 from rhodecode.lib.utils2 import LazyProperty, safe_str, safe_unicode,\
     remove_prefix, obfuscate_url_pw
 from rhodecode.lib.caching_query import FromCache
 from rhodecode.lib.hooks import log_create_repository, log_delete_repository
 
 from rhodecode.model import BaseModel
 from rhodecode.model.db import Repository, UserRepoToPerm, User, Permission, \
     Statistics, UserGroup, UserGroupRepoToPerm, RhodeCodeUi, RepoGroup,\
     RhodeCodeSetting, RepositoryField
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import HasRepoPermissionAny, HasUserGroupPermissionAny
 from rhodecode.lib.exceptions import AttachedForksError
 from rhodecode.model.scm import UserGroupList
 
 log = logging.getLogger(__name__)
 
 
 class RepoModel(BaseModel):
 
     cls = Repository
     URL_SEPARATOR = Repository.url_sep()
 
     def _get_user_group(self, users_group):
         return self._get_instance(UserGroup, users_group,
                                   callback=UserGroup.get_by_group_name)
 
     def _get_repo_group(self, repos_group):
         return self._get_instance(RepoGroup, repos_group,
                                   callback=RepoGroup.get_by_group_name)
 
     def _create_default_perms(self, repository, private):
         # create default permission
         default = 'repository.read'
         def_user = User.get_default_user()
         for p in def_user.user_perms:
             if p.permission.permission_name.startswith('repository.'):
                 default = p.permission.permission_name
                 break
 
         default_perm = 'repository.none' if private else default
 
         repo_to_perm = UserRepoToPerm()
         repo_to_perm.permission = Permission.get_by_key(default_perm)
 
         repo_to_perm.repository = repository
         repo_to_perm.user_id = def_user.user_id
 
         return repo_to_perm
 
     @LazyProperty
     def repos_path(self):
         """
         Get's the repositories root path from database
         """
 
         q = self.sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
         return q.ui_value
 
     def get(self, repo_id, cache=False):
         repo = self.sa.query(Repository)\
             .filter(Repository.repo_id == repo_id)
 
         if cache:
             repo = repo.options(FromCache("sql_cache_short",
                                           "get_repo_%s" % repo_id))
         return repo.scalar()
 
     def get_repo(self, repository):
         return self._get_repo(repository)
 
     def get_by_repo_name(self, repo_name, cache=False):
         repo = self.sa.query(Repository)\
             .filter(Repository.repo_name == repo_name)
 
         if cache:
             repo = repo.options(FromCache("sql_cache_short",
                                           "get_repo_%s" % repo_name))
         return repo.scalar()
 
     def get_all_user_repos(self, user):
         """
         Get's all repositories that user have at least read access
 
         :param user:
-        :type user:
         """
         from rhodecode.lib.auth import AuthUser
         user = self._get_user(user)
         repos = AuthUser(user_id=user.user_id).permissions['repositories']
         access_check = lambda r: r[1] in ['repository.read',
                                           'repository.write',
                                           'repository.admin']
         repos = [x[0] for x in filter(access_check, repos.items())]
         return Repository.query().filter(Repository.repo_name.in_(repos))
 
     def get_users_js(self):
         users = self.sa.query(User).filter(User.active == True).all()
         return json.dumps([
             {
              'id': u.user_id,
              'fname': u.name,
              'lname': u.lastname,
              'nname': u.username,
              'gravatar_lnk': h.gravatar_url(u.email, 14)
             } for u in users]
         )
 
     def get_users_groups_js(self):
         users_groups = self.sa.query(UserGroup)\
             .filter(UserGroup.users_group_active == True).all()
         users_groups = UserGroupList(users_groups, perm_set=['usergroup.read',
                                                              'usergroup.write',
                                                              'usergroup.admin'])
         return json.dumps([
             {
              'id': gr.users_group_id,
              'grname': gr.users_group_name,
              'grmembers': len(gr.members),
             } for gr in users_groups]
         )
 
     @classmethod
     def _render_datatable(cls, tmpl, *args, **kwargs):
         import rhodecode
         from pylons import tmpl_context as c
         from pylons.i18n.translation import _
 
         _tmpl_lookup = rhodecode.CONFIG['pylons.app_globals'].mako_lookup
         template = _tmpl_lookup.get_template('data_table/_dt_elements.html')
 
         tmpl = template.get_def(tmpl)
         kwargs.update(dict(_=_, h=h, c=c))
         return tmpl.render(*args, **kwargs)
 
     @classmethod
     def update_repoinfo(cls, repositories=None):
         if not repositories:
             repositories = Repository.getAll()
         for repo in repositories:
             repo.update_changeset_cache()
 
     def get_repos_as_dict(self, repos_list=None, admin=False, perm_check=True,
                           super_user_actions=False):
         _render = self._render_datatable
         from pylons import tmpl_context as c
 
         def quick_menu(repo_name):
             return _render('quick_menu', repo_name)
 
         def repo_lnk(name, rtype, private, fork_of):
             return _render('repo_name', name, rtype, private, fork_of,
                            short_name=not admin, admin=False)
 
         def last_change(last_change):
             return _render("last_change", last_change)
 
         def rss_lnk(repo_name):
             return _render("rss", repo_name)
 
         def atom_lnk(repo_name):
             return _render("atom", repo_name)
 
         def last_rev(repo_name, cs_cache):
             return _render('revision', repo_name, cs_cache.get('revision'),
                            cs_cache.get('raw_id'), cs_cache.get('author'),
                            cs_cache.get('message'))
 
         def desc(desc):
             if c.visual.stylify_metatags:
                 return h.urlify_text(h.desc_stylize(h.truncate(desc, 60)))
             else:
                 return h.urlify_text(h.truncate(desc, 60))
 
         def repo_actions(repo_name):
             return _render('repo_actions', repo_name, super_user_actions)
 
         def owner_actions(user_id, username):
             return _render('user_name', user_id, username)
 
         repos_data = []
         for repo in repos_list:
             if perm_check:
                 # check permission at this level
                 if not HasRepoPermissionAny(
                     'repository.read', 'repository.write', 'repository.admin'
                 )(repo.repo_name, 'get_repos_as_dict check'):
                     continue
             cs_cache = repo.changeset_cache
             row = {
                 "menu": quick_menu(repo.repo_name),
                 "raw_name": repo.repo_name.lower(),
                 "name": repo_lnk(repo.repo_name, repo.repo_type,
                                  repo.private, repo.fork),
                 "last_change": last_change(repo.last_db_change),
                 "last_changeset": last_rev(repo.repo_name, cs_cache),
                 "raw_tip": cs_cache.get('revision'),
                 "desc": desc(repo.description),
                 "owner": h.person(repo.user.username),
                 "rss": rss_lnk(repo.repo_name),
                 "atom": atom_lnk(repo.repo_name),
 
             }
             if admin:
                 row.update({
                     "action": repo_actions(repo.repo_name),
                     "owner": owner_actions(repo.user.user_id,
                                            h.person(repo.user.username))
                 })
             repos_data.append(row)
 
         return {
             "totalRecords": len(repos_list),
             "startIndex": 0,
             "sort": "name",
             "dir": "asc",
             "records": repos_data
         }
 
     def _get_defaults(self, repo_name):
         """
         Get's information about repository, and returns a dict for
         usage in forms
 
         :param repo_name:
         """
 
         repo_info = Repository.get_by_repo_name(repo_name)
 
         if repo_info is None:
             return None
 
         defaults = repo_info.get_dict()
         group, repo_name, repo_name_full = repo_info.groups_and_repo
         defaults['repo_name'] = repo_name
         defaults['repo_group'] = getattr(group[-1] if group else None,
                                          'group_id', None)
 
         for strip, k in [(0, 'repo_type'), (1, 'repo_enable_downloads'),
                   (1, 'repo_description'), (1, 'repo_enable_locking'),
                   (1, 'repo_landing_rev'), (0, 'clone_uri'),
                   (1, 'repo_private'), (1, 'repo_enable_statistics')]:
             attr = k
             if strip:
                 attr = remove_prefix(k, 'repo_')
 
             defaults[k] = defaults[attr]
 
         # fill owner
         if repo_info.user:
             defaults.update({'user': repo_info.user.username})
         else:
             replacement_user = User.query().filter(User.admin ==
                                                    True).first().username
             defaults.update({'user': replacement_user})
 
         # fill repository users
         for p in repo_info.repo_to_perm:
             defaults.update({'u_perm_%s' % p.user.username:
                              p.permission.permission_name})
 
         # fill repository groups
         for p in repo_info.users_group_to_perm:
             defaults.update({'g_perm_%s' % p.users_group.users_group_name:
                              p.permission.permission_name})
 
         return defaults
 
     def update(self, org_repo_name, **kwargs):
         try:
             cur_repo = self.get_by_repo_name(org_repo_name, cache=False)
 
             if 'user' in kwargs:
                 cur_repo.user = User.get_by_username(kwargs['user'])
 
             if 'repo_group' in kwargs:
                 cur_repo.group = RepoGroup.get(kwargs['repo_group'])
 
             for strip, k in [(0, 'repo_type'), (1, 'repo_enable_downloads'),
                       (1, 'repo_description'), (1, 'repo_enable_locking'),
                       (1, 'repo_landing_rev'), (0, 'clone_uri'),
                       (1, 'repo_private'), (1, 'repo_enable_statistics')]:
                 if k in kwargs:
                     val = kwargs[k]
                     if strip:
                         k = remove_prefix(k, 'repo_')
                     setattr(cur_repo, k, val)
 
             new_name = cur_repo.get_new_name(kwargs['repo_name'])
             cur_repo.repo_name = new_name
             #if private flag is set, reset default permission to NONE
 
             if kwargs.get('repo_private'):
                 EMPTY_PERM = 'repository.none'
                 RepoModel().grant_user_permission(
                     repo=cur_repo, user='default', perm=EMPTY_PERM
                 )
             #handle extra fields
             for field in filter(lambda k: k.startswith(RepositoryField.PREFIX), kwargs):
                 k = RepositoryField.un_prefix_key(field)
                 ex_field = RepositoryField.get_by_key_name(key=k, repo=cur_repo)
                 if ex_field:
                     ex_field.field_value = kwargs[field]
                     self.sa.add(ex_field)
             self.sa.add(cur_repo)
 
             if org_repo_name != new_name:
                 # rename repository
                 self.__rename_repo(old=org_repo_name, new=new_name)
 
             return cur_repo
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def create_repo(self, repo_name, repo_type, description, owner,
                     private=False, clone_uri=None, repos_group=None,
                     landing_rev='tip', just_db=False, fork_of=None,
                     copy_fork_permissions=False, enable_statistics=False,
                     enable_locking=False, enable_downloads=False):
         """
         Create repository
 
         """
         from rhodecode.model.scm import ScmModel
 
         owner = self._get_user(owner)
         fork_of = self._get_repo(fork_of)
         repos_group = self._get_repo_group(repos_group)
         try:
 
             # repo name is just a name of repository
             # while repo_name_full is a full qualified name that is combined
             # with name and path of group
             repo_name_full = repo_name
             repo_name = repo_name.split(self.URL_SEPARATOR)[-1]
 
             new_repo = Repository()
             new_repo.enable_statistics = False
             new_repo.repo_name = repo_name_full
             new_repo.repo_type = repo_type
             new_repo.user = owner
             new_repo.group = repos_group
             new_repo.description = description or repo_name
             new_repo.private = private
             new_repo.clone_uri = clone_uri
             new_repo.landing_rev = landing_rev
 
             new_repo.enable_statistics = enable_statistics
             new_repo.enable_locking = enable_locking
             new_repo.enable_downloads = enable_downloads
 
             if repos_group:
                 new_repo.enable_locking = repos_group.enable_locking
 
             if fork_of:
                 parent_repo = fork_of
                 new_repo.fork = parent_repo
 
             self.sa.add(new_repo)
 
             if fork_of:
                 if copy_fork_permissions:
                     repo = fork_of
                     user_perms = UserRepoToPerm.query()\
                         .filter(UserRepoToPerm.repository == repo).all()
                     group_perms = UserGroupRepoToPerm.query()\
                         .filter(UserGroupRepoToPerm.repository == repo).all()
 
                     for perm in user_perms:
                         UserRepoToPerm.create(perm.user, new_repo,
                                               perm.permission)
 
                     for perm in group_perms:
                         UserGroupRepoToPerm.create(perm.users_group, new_repo,
                                                     perm.permission)
                 else:
                     perm_obj = self._create_default_perms(new_repo, private)
                     self.sa.add(perm_obj)
             else:
                 perm_obj = self._create_default_perms(new_repo, private)
                 self.sa.add(perm_obj)
 
             if not just_db:
                 self.__create_repo(repo_name, repo_type,
                                    repos_group,
                                    clone_uri)
                 log_create_repository(new_repo.get_dict(),
                                       created_by=owner.username)
 
             # now automatically start following this repository as owner
             ScmModel(self.sa).toggle_following_repo(new_repo.repo_id,
                                                     owner.user_id)
             return new_repo
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def create(self, form_data, cur_user, just_db=False, fork=None):
         """
         Backward compatibility function, just a wrapper on top of create_repo
 
         :param form_data:
         :param cur_user:
         :param just_db:
         :param fork:
         """
         owner = cur_user
         repo_name = form_data['repo_name_full']
         repo_type = form_data['repo_type']
         description = form_data['repo_description']
         private = form_data['repo_private']
         clone_uri = form_data.get('clone_uri')
         repos_group = form_data['repo_group']
         landing_rev = form_data['repo_landing_rev']
         copy_fork_permissions = form_data.get('copy_permissions')
         fork_of = form_data.get('fork_parent_id')
 
         ## repo creation defaults, private and repo_type are filled in form
         defs = RhodeCodeSetting.get_default_repo_settings(strip_prefix=True)
         enable_statistics = defs.get('repo_enable_statistics')
         enable_locking = defs.get('repo_enable_locking')
         enable_downloads = defs.get('repo_enable_downloads')
 
         return self.create_repo(
             repo_name, repo_type, description, owner, private, clone_uri,
             repos_group, landing_rev, just_db, fork_of, copy_fork_permissions,
             enable_statistics, enable_locking, enable_downloads
         )
 
     def _update_permissions(self, repo, perms_new=None, perms_updates=None,
                             check_perms=True):
         if not perms_new:
             perms_new = []
         if not perms_updates:
             perms_updates = []
 
         # update permissions
         for member, perm, member_type in perms_updates:
             if member_type == 'user':
                 # this updates existing one
                 self.grant_user_permission(
                     repo=repo, user=member, perm=perm
                 )
             else:
                 #check if we have permissions to alter this usergroup
                 req_perms = ('usergroup.read', 'usergroup.write', 'usergroup.admin')
                 if not check_perms or HasUserGroupPermissionAny(*req_perms)(member):
                     self.grant_users_group_permission(
                         repo=repo, group_name=member, perm=perm
                     )
         # set new permissions
         for member, perm, member_type in perms_new:
             if member_type == 'user':
                 self.grant_user_permission(
                     repo=repo, user=member, perm=perm
                 )
             else:
                 #check if we have permissions to alter this usergroup
                 req_perms = ('usergroup.read', 'usergroup.write', 'usergroup.admin')
                 if not check_perms or HasUserGroupPermissionAny(*req_perms)(member):
                     self.grant_users_group_permission(
                         repo=repo, group_name=member, perm=perm
                     )
 
     def create_fork(self, form_data, cur_user):
         """
         Simple wrapper into executing celery task for fork creation
 
         :param form_data:
         :param cur_user:
         """
         from rhodecode.lib.celerylib import tasks, run_task
         run_task(tasks.create_repo_fork, form_data, cur_user)
 
     def delete(self, repo, forks=None, fs_remove=True):
         """
         Delete given repository, forks parameter defines what do do with
         attached forks. Throws AttachedForksError if deleted repo has attached
         forks
 
         :param repo:
         :param forks: str 'delete' or 'detach'
         :param fs_remove: remove(archive) repo from filesystem
         """
         repo = self._get_repo(repo)
         if repo:
             if forks == 'detach':
                 for r in repo.forks:
                     r.fork = None
                     self.sa.add(r)
             elif forks == 'delete':
                 for r in repo.forks:
                     self.delete(r, forks='delete')
             elif [f for f in repo.forks]:
                 raise AttachedForksError()
 
             old_repo_dict = repo.get_dict()
             owner = repo.user
             try:
                 self.sa.delete(repo)
                 if fs_remove:
                     self.__delete_repo(repo)
                 else:
                     log.debug('skipping removal from filesystem')
                 log_delete_repository(old_repo_dict,
                                       deleted_by=owner.username)
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
     def grant_user_permission(self, repo, user, perm):
         """
         Grant permission for user on given repository, or update existing one
         if found
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param user: Instance of User, user_id or username
         :param perm: Instance of Permission, or permission_name
         """
         user = self._get_user(user)
         repo = self._get_repo(repo)
         permission = self._get_perm(perm)
 
         # check if we have that permission already
         obj = self.sa.query(UserRepoToPerm)\
             .filter(UserRepoToPerm.user == user)\
             .filter(UserRepoToPerm.repository == repo)\
             .scalar()
         if obj is None:
             # create new !
             obj = UserRepoToPerm()
         obj.repository = repo
         obj.user = user
         obj.permission = permission
         self.sa.add(obj)
         log.debug('Granted perm %s to %s on %s' % (perm, user, repo))
 
     def revoke_user_permission(self, repo, user):
         """
         Revoke permission for user on given repository
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param user: Instance of User, user_id or username
         """
 
         user = self._get_user(user)
         repo = self._get_repo(repo)
 
         obj = self.sa.query(UserRepoToPerm)\
             .filter(UserRepoToPerm.repository == repo)\
             .filter(UserRepoToPerm.user == user)\
             .scalar()
         if obj:
             self.sa.delete(obj)
             log.debug('Revoked perm on %s on %s' % (repo, user))
 
     def grant_users_group_permission(self, repo, group_name, perm):
         """
         Grant permission for user group on given repository, or update
         existing one if found
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         :param perm: Instance of Permission, or permission_name
         """
         repo = self._get_repo(repo)
         group_name = self._get_user_group(group_name)
         permission = self._get_perm(perm)
 
         # check if we have that permission already
         obj = self.sa.query(UserGroupRepoToPerm)\
             .filter(UserGroupRepoToPerm.users_group == group_name)\
             .filter(UserGroupRepoToPerm.repository == repo)\
             .scalar()
 
         if obj is None:
             # create new
             obj = UserGroupRepoToPerm()
 
         obj.repository = repo
         obj.users_group = group_name
         obj.permission = permission
         self.sa.add(obj)
         log.debug('Granted perm %s to %s on %s' % (perm, group_name, repo))
 
     def revoke_users_group_permission(self, repo, group_name):
         """
         Revoke permission for user group on given repository
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         """
         repo = self._get_repo(repo)
         group_name = self._get_user_group(group_name)
 
         obj = self.sa.query(UserGroupRepoToPerm)\
             .filter(UserGroupRepoToPerm.repository == repo)\
             .filter(UserGroupRepoToPerm.users_group == group_name)\
             .scalar()
         if obj:
             self.sa.delete(obj)
             log.debug('Revoked perm to %s on %s' % (repo, group_name))
 
     def delete_stats(self, repo_name):
         """
         removes stats for given repo
 
         :param repo_name:
         """
         repo = self._get_repo(repo_name)
         try:
             obj = self.sa.query(Statistics)\
                     .filter(Statistics.repository == repo).scalar()
             if obj:
                 self.sa.delete(obj)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
-    def __create_repo(self, repo_name, alias, parent, clone_uri=False):
+    def _create_repo(self, repo_name, alias, parent, clone_uri=False,
+                     repo_store_location=None):
+        return self.__create_repo(repo_name, alias, parent, clone_uri,
+                                  repo_store_location)
+
+    def __create_repo(self, repo_name, alias, parent, clone_uri=False,
+                      repo_store_location=None):
         """
         makes repository on filesystem. It's group aware means it'll create
         a repository within a group, and alter the paths accordingly of
         group location
 
         :param repo_name:
         :param alias:
         :param parent_id:
         :param clone_uri:
+        :param repo_path:
         """
         from rhodecode.lib.utils import is_valid_repo, is_valid_repos_group
         from rhodecode.model.scm import ScmModel
 
         if parent:
             new_parent_path = os.sep.join(parent.full_path_splitted)
         else:
             new_parent_path = ''
-
+        if repo_store_location:
+            _paths = [repo_store_location]
+        else:
+            _paths = [self.repos_path, new_parent_path, repo_name]
         # we need to make it str for mercurial
-        repo_path = os.path.join(*map(lambda x: safe_str(x),
-                                [self.repos_path, new_parent_path, repo_name]))
+        repo_path = os.path.join(*map(lambda x: safe_str(x), _paths))
 
         # check if this path is not a repository
         if is_valid_repo(repo_path, self.repos_path):
             raise Exception('This path %s is a valid repository' % repo_path)
 
         # check if this path is a group
         if is_valid_repos_group(repo_path, self.repos_path):
             raise Exception('This path %s is a valid group' % repo_path)
 
         log.info('creating repo %s in %s @ %s' % (
                      repo_name, safe_unicode(repo_path),
                      obfuscate_url_pw(clone_uri)
                 )
         )
         backend = get_backend(alias)
         if alias == 'hg':
-            backend(repo_path, create=True, src_url=clone_uri)
+            repo = backend(repo_path, create=True, src_url=clone_uri)
         elif alias == 'git':
-            r = backend(repo_path, create=True, src_url=clone_uri, bare=True)
+            repo = backend(repo_path, create=True, src_url=clone_uri, bare=True)
             # add rhodecode hook into this repo
-            ScmModel().install_git_hook(repo=r)
+            ScmModel().install_git_hook(repo=repo)
         else:
             raise Exception('Undefined alias %s' % alias)
+        return repo
 
     def __rename_repo(self, old, new):
         """
         renames repository on filesystem
 
         :param old: old name
         :param new: new name
         """
         log.info('renaming repo from %s to %s' % (old, new))
 
         old_path = os.path.join(self.repos_path, old)
         new_path = os.path.join(self.repos_path, new)
         if os.path.isdir(new_path):
             raise Exception(
                 'Was trying to rename to already existing dir %s' % new_path
             )
         shutil.move(old_path, new_path)
 
     def __delete_repo(self, repo):
         """
         removes repo from filesystem, the removal is acctually made by
         added rm__ prefix into dir, and rename internat .hg/.git dirs so this
         repository is no longer valid for rhodecode, can be undeleted later on
         by reverting the renames on this repository
 
         :param repo: repo object
         """
         rm_path = os.path.join(self.repos_path, repo.repo_name)
         log.info("Removing %s" % (rm_path))
         # disable hg/git internal that it doesn't get detected as repo
         alias = repo.repo_type
 
         bare = getattr(repo.scm_instance, 'bare', False)
 
         if not bare:
             # skip this for bare git repos
             shutil.move(os.path.join(rm_path, '.%s' % alias),
                         os.path.join(rm_path, 'rm__.%s' % alias))
         # disable repo
         _now = datetime.now()
         _ms = str(_now.microsecond).rjust(6, '0')
         _d = 'rm__%s__%s' % (_now.strftime('%Y%m%d_%H%M%S_' + _ms),
                              repo.just_name)
         if repo.group:
             args = repo.group.full_path_splitted + [_d]
             _d = os.path.join(*args)
         shutil.move(rm_path, os.path.join(self.repos_path, _d))

rhodecode/model/scm.py

@@ -1,695 +1,727 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.model.scm
     ~~~~~~~~~~~~~~~~~~~
 
     Scm model for RhodeCode
 
     :created_on: Apr 9, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import with_statement
 import os
 import re
 import time
 import traceback
 import logging
 import cStringIO
 import pkg_resources
 from os.path import dirname as dn, join as jn
 
 from sqlalchemy import func
 from pylons.i18n.translation import _
 
 import rhodecode
 from rhodecode.lib.vcs import get_backend
 from rhodecode.lib.vcs.exceptions import RepositoryError
 from rhodecode.lib.vcs.utils.lazy import LazyProperty
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.lib.vcs.backends.base import EmptyChangeset
 
 from rhodecode import BACKENDS
 from rhodecode.lib import helpers as h
 from rhodecode.lib.utils2 import safe_str, safe_unicode, get_server_url,\
     _set_extras
 from rhodecode.lib.auth import HasRepoPermissionAny, HasReposGroupPermissionAny,\
     HasUserGroupPermissionAnyDecorator, HasUserGroupPermissionAny
 from rhodecode.lib.utils import get_filesystem_repos, make_ui, \
     action_logger, REMOVED_REPO_PAT
 from rhodecode.model import BaseModel
 from rhodecode.model.db import Repository, RhodeCodeUi, CacheInvalidation, \
     UserFollowing, UserLog, User, RepoGroup, PullRequest
 from rhodecode.lib.hooks import log_push_action
+from rhodecode.lib.exceptions import NonRelativePathError
 
 log = logging.getLogger(__name__)
 
 
 class UserTemp(object):
     def __init__(self, user_id):
         self.user_id = user_id
 
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id)
 
 
 class RepoTemp(object):
     def __init__(self, repo_id):
         self.repo_id = repo_id
 
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id)
 
 
 class CachedRepoList(object):
     """
     Cached repo list, uses in-memory cache after initialization, that is
     super fast
     """
 
     def __init__(self, db_repo_list, repos_path, order_by=None, perm_set=None):
         self.db_repo_list = db_repo_list
         self.repos_path = repos_path
         self.order_by = order_by
         self.reversed = (order_by or '').startswith('-')
         if not perm_set:
             perm_set = ['repository.read', 'repository.write',
                         'repository.admin']
         self.perm_set = perm_set
 
     def __len__(self):
         return len(self.db_repo_list)
 
     def __repr__(self):
         return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
 
     def __iter__(self):
         # pre-propagated valid_cache_keys to save executing select statements
         # for each repo
         valid_cache_keys = CacheInvalidation.get_valid_cache_keys()
 
         for dbr in self.db_repo_list:
             scmr = dbr.scm_instance_cached(valid_cache_keys)
             # check permission at this level
             if not HasRepoPermissionAny(
                 *self.perm_set)(dbr.repo_name, 'get repo check'):
                 continue
 
             try:
                 last_change = scmr.last_change
                 tip = h.get_changeset_safe(scmr, 'tip')
             except Exception:
                 log.error(
                     '%s this repository is present in database but it '
                     'cannot be created as an scm instance, org_exc:%s'
                     % (dbr.repo_name, traceback.format_exc())
                 )
                 continue
 
             tmp_d = {}
             tmp_d['name'] = dbr.repo_name
             tmp_d['name_sort'] = tmp_d['name'].lower()
             tmp_d['raw_name'] = tmp_d['name'].lower()
             tmp_d['description'] = dbr.description
             tmp_d['description_sort'] = tmp_d['description'].lower()
             tmp_d['last_change'] = last_change
             tmp_d['last_change_sort'] = time.mktime(last_change.timetuple())
             tmp_d['tip'] = tip.raw_id
             tmp_d['tip_sort'] = tip.revision
             tmp_d['rev'] = tip.revision
             tmp_d['contact'] = dbr.user.full_contact
             tmp_d['contact_sort'] = tmp_d['contact']
             tmp_d['owner_sort'] = tmp_d['contact']
             tmp_d['repo_archives'] = list(scmr._get_archives())
             tmp_d['last_msg'] = tip.message
             tmp_d['author'] = tip.author
             tmp_d['dbrepo'] = dbr.get_dict()
             tmp_d['dbrepo_fork'] = dbr.fork.get_dict() if dbr.fork else {}
             yield tmp_d
 
 
 class SimpleCachedRepoList(CachedRepoList):
     """
     Lighter version of CachedRepoList without the scm initialisation
     """
 
     def __iter__(self):
         for dbr in self.db_repo_list:
             # check permission at this level
             if not HasRepoPermissionAny(
                 *self.perm_set)(dbr.repo_name, 'get repo check'):
                 continue
 
             tmp_d = {}
             tmp_d['name'] = dbr.repo_name
             tmp_d['name_sort'] = tmp_d['name'].lower()
             tmp_d['raw_name'] = tmp_d['name'].lower()
             tmp_d['description'] = dbr.description
             tmp_d['description_sort'] = tmp_d['description'].lower()
             tmp_d['dbrepo'] = dbr.get_dict()
             tmp_d['dbrepo_fork'] = dbr.fork.get_dict() if dbr.fork else {}
             yield tmp_d
 
 
 class _PermCheckIterator(object):
     def __init__(self, obj_list, obj_attr, perm_set, perm_checker):
         """
         Creates iterator from given list of objects, additionally
         checking permission for them from perm_set var
 
         :param obj_list: list of db objects
         :param obj_attr: attribute of object to pass into perm_checker
         :param perm_set: list of permissions to check
         :param perm_checker: callable to check permissions against
         """
         self.obj_list = obj_list
         self.obj_attr = obj_attr
         self.perm_set = perm_set
         self.perm_checker = perm_checker
 
     def __len__(self):
         return len(self.obj_list)
 
     def __repr__(self):
         return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
 
     def __iter__(self):
         for db_obj in self.obj_list:
             # check permission at this level
             name = getattr(db_obj, self.obj_attr, None)
             if not self.perm_checker(*self.perm_set)(name, self.__class__.__name__):
                 continue
 
             yield db_obj
 
 
 class RepoGroupList(_PermCheckIterator):
 
     def __init__(self, db_repo_group_list, perm_set=None):
         if not perm_set:
             perm_set = ['group.read', 'group.write', 'group.admin']
 
         super(RepoGroupList, self).__init__(obj_list=db_repo_group_list,
                     obj_attr='group_name', perm_set=perm_set,
                     perm_checker=HasReposGroupPermissionAny)
 
 
 class UserGroupList(_PermCheckIterator):
 
     def __init__(self, db_user_group_list, perm_set=None):
         if not perm_set:
             perm_set = ['usergroup.read', 'usergroup.write', 'usergroup.admin']
 
         super(UserGroupList, self).__init__(obj_list=db_user_group_list,
                     obj_attr='users_group_name', perm_set=perm_set,
                     perm_checker=HasUserGroupPermissionAny)
 
 
 class ScmModel(BaseModel):
     """
     Generic Scm Model
     """
 
     def __get_repo(self, instance):
         cls = Repository
         if isinstance(instance, cls):
             return instance
         elif isinstance(instance, int) or safe_str(instance).isdigit():
             return cls.get(instance)
         elif isinstance(instance, basestring):
             return cls.get_by_repo_name(instance)
         elif instance:
             raise Exception('given object must be int, basestr or Instance'
                             ' of %s got %s' % (type(cls), type(instance)))
 
     @LazyProperty
     def repos_path(self):
         """
         Get's the repositories root path from database
         """
 
         q = self.sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
 
         return q.ui_value
 
     def repo_scan(self, repos_path=None):
         """
         Listing of repositories in given path. This path should not be a
         repository itself. Return a dictionary of repository objects
 
         :param repos_path: path to directory containing repositories
         """
 
         if repos_path is None:
             repos_path = self.repos_path
 
         log.info('scanning for repositories in %s' % repos_path)
 
         baseui = make_ui('db')
         repos = {}
 
         for name, path in get_filesystem_repos(repos_path, recursive=True):
             # name need to be decomposed and put back together using the /
             # since this is internal storage separator for rhodecode
             name = Repository.normalize_repo_name(name)
 
             try:
                 if name in repos:
                     raise RepositoryError('Duplicate repository name %s '
                                           'found in %s' % (name, path))
                 else:
 
                     klass = get_backend(path[0])
 
                     if path[0] == 'hg' and path[0] in BACKENDS.keys():
                         repos[name] = klass(safe_str(path[1]), baseui=baseui)
 
                     if path[0] == 'git' and path[0] in BACKENDS.keys():
                         repos[name] = klass(path[1])
             except OSError:
                 continue
         log.debug('found %s paths with repositories' % (len(repos)))
         return repos
 
     def get_repos(self, all_repos=None, sort_key=None, simple=False):
         """
         Get all repos from db and for each repo create it's
         backend instance and fill that backed with information from database
 
         :param all_repos: list of repository names as strings
             give specific repositories list, good for filtering
 
         :param sort_key: initial sorting of repos
         :param simple: use SimpleCachedList - one without the SCM info
         """
         if all_repos is None:
             all_repos = self.sa.query(Repository)\
                         .filter(Repository.group_id == None)\
                         .order_by(func.lower(Repository.repo_name)).all()
         if simple:
             repo_iter = SimpleCachedRepoList(all_repos,
                                              repos_path=self.repos_path,
                                              order_by=sort_key)
         else:
             repo_iter = CachedRepoList(all_repos,
                                        repos_path=self.repos_path,
                                        order_by=sort_key)
 
         return repo_iter
 
     def get_repos_groups(self, all_groups=None):
         if all_groups is None:
             all_groups = RepoGroup.query()\
                 .filter(RepoGroup.group_parent_id == None).all()
         return [x for x in RepoGroupList(all_groups)]
 
     def mark_for_invalidation(self, repo_name):
         """
         Mark caches of this repo invalid in the database.
 
         :param repo_name: the repo for which caches should be marked invalid
         """
         CacheInvalidation.set_invalidate(repo_name)
         repo = Repository.get_by_repo_name(repo_name)
         if repo:
             repo.update_changeset_cache()
 
     def toggle_following_repo(self, follow_repo_id, user_id):
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_repo_id == follow_repo_id)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         if f is not None:
             try:
                 self.sa.delete(f)
                 action_logger(UserTemp(user_id),
                               'stopped_following_repo',
                               RepoTemp(follow_repo_id))
                 return
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_repo_id = follow_repo_id
             self.sa.add(f)
 
             action_logger(UserTemp(user_id),
                           'started_following_repo',
                           RepoTemp(follow_repo_id))
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def toggle_following_user(self, follow_user_id, user_id):
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_user_id == follow_user_id)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         if f is not None:
             try:
                 self.sa.delete(f)
                 return
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_user_id = follow_user_id
             self.sa.add(f)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def is_following_repo(self, repo_name, user_id, cache=False):
         r = self.sa.query(Repository)\
             .filter(Repository.repo_name == repo_name).scalar()
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_repository == r)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         return f is not None
 
     def is_following_user(self, username, user_id, cache=False):
         u = User.get_by_username(username)
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_user == u)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         return f is not None
 
     def get_followers(self, repo):
         repo = self._get_repo(repo)
 
         return self.sa.query(UserFollowing)\
                 .filter(UserFollowing.follows_repository == repo).count()
 
     def get_forks(self, repo):
         repo = self._get_repo(repo)
         return self.sa.query(Repository)\
                 .filter(Repository.fork == repo).count()
 
     def get_pull_requests(self, repo):
         repo = self._get_repo(repo)
         return self.sa.query(PullRequest)\
                 .filter(PullRequest.other_repo == repo)\
                 .filter(PullRequest.status != PullRequest.STATUS_CLOSED).count()
 
     def mark_as_fork(self, repo, fork, user):
         repo = self.__get_repo(repo)
         fork = self.__get_repo(fork)
         if fork and repo.repo_id == fork.repo_id:
             raise Exception("Cannot set repository as fork of itself")
         repo.fork = fork
         self.sa.add(repo)
         return repo
 
     def _handle_rc_scm_extras(self, username, repo_name, repo_alias):
         from rhodecode import CONFIG
         from rhodecode.lib.base import _get_ip_addr
         try:
             from pylons import request
             environ = request.environ
         except TypeError:
             # we might use this outside of request context, let's fake the
             # environ data
             from webob import Request
             environ = Request.blank('').environ
         extras = {
             'ip': _get_ip_addr(environ),
             'username': username,
             'action': 'push_local',
             'repository': repo_name,
             'scm': repo_alias,
             'config': CONFIG['__file__'],
             'server_url': get_server_url(environ),
             'make_lock': None,
             'locked_by': [None, None]
         }
         _set_extras(extras)
 
     def _handle_push(self, repo, username, action, repo_name, revisions):
         """
         Triggers push action hooks
 
         :param repo: SCM repo
         :param username: username who pushes
         :param action: push/push_loca/push_remote
         :param repo_name: name of repo
         :param revisions: list of revisions that we pushed
         """
         self._handle_rc_scm_extras(username, repo_name, repo_alias=repo.alias)
         _scm_repo = repo._repo
         # trigger push hook
         if repo.alias == 'hg':
             log_push_action(_scm_repo.ui, _scm_repo, node=revisions[0])
         elif repo.alias == 'git':
             log_push_action(None, _scm_repo, _git_revs=revisions)
 
     def _get_IMC_module(self, scm_type):
         """
         Returns InMemoryCommit class based on scm_type
 
         :param scm_type:
         """
         if scm_type == 'hg':
             from rhodecode.lib.vcs.backends.hg import \
                 MercurialInMemoryChangeset as IMC
         elif scm_type == 'git':
             from rhodecode.lib.vcs.backends.git import \
                 GitInMemoryChangeset as IMC
         return IMC
 
     def pull_changes(self, repo, username):
         dbrepo = self.__get_repo(repo)
         clone_uri = dbrepo.clone_uri
         if not clone_uri:
             raise Exception("This repository doesn't have a clone uri")
 
         repo = dbrepo.scm_instance
         repo_name = dbrepo.repo_name
         try:
             if repo.alias == 'git':
                 repo.fetch(clone_uri)
             else:
                 repo.pull(clone_uri)
             self.mark_for_invalidation(repo_name)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def commit_change(self, repo, repo_name, cs, user, author, message,
                       content, f_path):
         """
         Commits changes
 
         :param repo: SCM instance
 
         """
         user = self._get_user(user)
         IMC = self._get_IMC_module(repo.alias)
 
         # decoding here will force that we have proper encoded values
         # in any other case this will throw exceptions and deny commit
         content = safe_str(content)
         path = safe_str(f_path)
         # message and author needs to be unicode
         # proper backend should then translate that into required type
         message = safe_unicode(message)
         author = safe_unicode(author)
         imc = IMC(repo)
         imc.change(FileNode(path, content, mode=cs.get_file_mode(f_path)))
         tip = imc.commit(message=message,
                        author=author,
                        parents=[cs], branch=cs.branch)
 
         self.mark_for_invalidation(repo_name)
         self._handle_push(repo,
                           username=user.username,
                           action='push_local',
                           repo_name=repo_name,
                           revisions=[tip.raw_id])
         return tip
 
-    def create_node(self, repo, repo_name, cs, user, author, message, content,
-                      f_path):
+    def create_nodes(self, user, repo, message, nodes, parent_cs=None,
+                     author=None, trigger_push_hook=True):
+        """
+        Commits given multiple nodes into repo
+
+        :param user: RhodeCode User object or user_id, the commiter
+        :param repo: RhodeCode Repository object
+        :param message: commit message
+        :param nodes: mapping {filename:{'content':content},...}
+        :param parent_cs: parent changeset, can be empty than it's initial commit
+        :param author: author of commit, cna be different that commiter only for git
+        :param trigger_push_hook: trigger push hooks
+
+        :returns: new commited changeset
+        """
+
         user = self._get_user(user)
-        IMC = self._get_IMC_module(repo.alias)
+        scm_instance = repo.scm_instance_no_cache()
 
-        # decoding here will force that we have proper encoded values
-        # in any other case this will throw exceptions and deny commit
-        if isinstance(content, (basestring,)):
-            content = safe_str(content)
-        elif isinstance(content, (file, cStringIO.OutputType,)):
-            content = content.read()
-        else:
-            raise Exception('Content is of unrecognized type %s' % (
-                type(content)
-            ))
+        processed_nodes = []
+        for f_path in nodes:
+            if f_path.startswith('/') or f_path.startswith('.') or '../' in f_path:
+                raise NonRelativePathError('%s is not an relative path' % f_path)
+            if f_path:
+                f_path = os.path.normpath(f_path)
+            f_path = safe_str(f_path)
+            content = nodes[f_path]['content']
+            # decoding here will force that we have proper encoded values
+            # in any other case this will throw exceptions and deny commit
+            if isinstance(content, (basestring,)):
+                content = safe_str(content)
+            elif isinstance(content, (file, cStringIO.OutputType,)):
+                content = content.read()
+            else:
+                raise Exception('Content is of unrecognized type %s' % (
+                    type(content)
+                ))
+            processed_nodes.append((f_path, content))
 
         message = safe_unicode(message)
-        author = safe_unicode(author)
-        path = safe_str(f_path)
-        m = IMC(repo)
+        commiter = user.full_contact
+        author = safe_unicode(author) if author else commiter
 
-        if isinstance(cs, EmptyChangeset):
+        IMC = self._get_IMC_module(scm_instance.alias)
+        imc = IMC(scm_instance)
+
+        if not parent_cs:
+            parent_cs = EmptyChangeset(alias=scm_instance.alias)
+
+        if isinstance(parent_cs, EmptyChangeset):
             # EmptyChangeset means we we're editing empty repository
             parents = None
         else:
-            parents = [cs]
-
-        m.add(FileNode(path, content=content))
-        tip = m.commit(message=message,
-                       author=author,
-                       parents=parents, branch=cs.branch)
+            parents = [parent_cs]
+        # add multiple nodes
+        for path, content in processed_nodes:
+            imc.add(FileNode(path, content=content))
 
-        self.mark_for_invalidation(repo_name)
-        self._handle_push(repo,
-                          username=user.username,
-                          action='push_local',
-                          repo_name=repo_name,
-                          revisions=[tip.raw_id])
+        tip = imc.commit(message=message,
+                         author=author,
+                         parents=parents,
+                         branch=parent_cs.branch)
+
+        self.mark_for_invalidation(repo.repo_name)
+        if trigger_push_hook:
+            self._handle_push(scm_instance,
+                              username=user.username,
+                              action='push_local',
+                              repo_name=repo.repo_name,
+                              revisions=[tip.raw_id])
         return tip
 
     def get_nodes(self, repo_name, revision, root_path='/', flat=True):
         """
         recursive walk in root dir and return a set of all path in that dir
         based on repository walk function
 
         :param repo_name: name of repository
         :param revision: revision for which to list nodes
         :param root_path: root path to list
         :param flat: return as a list, if False returns a dict with decription
 
         """
         _files = list()
         _dirs = list()
         try:
             _repo = self.__get_repo(repo_name)
             changeset = _repo.scm_instance.get_changeset(revision)
             root_path = root_path.lstrip('/')
             for topnode, dirs, files in changeset.walk(root_path):
                 for f in files:
                     _files.append(f.path if flat else {"name": f.path,
                                                        "type": "file"})
                 for d in dirs:
                     _dirs.append(d.path if flat else {"name": d.path,
                                                       "type": "dir"})
         except RepositoryError:
             log.debug(traceback.format_exc())
             raise
 
         return _dirs, _files
 
     def get_unread_journal(self):
         return self.sa.query(UserLog).count()
 
     def get_repo_landing_revs(self, repo=None):
         """
         Generates select option with tags branches and bookmarks (for hg only)
         grouped by type
 
         :param repo:
-        :type repo:
         """
 
         hist_l = []
         choices = []
         repo = self.__get_repo(repo)
         hist_l.append(['tip', _('latest tip')])
         choices.append('tip')
         if not repo:
             return choices, hist_l
 
         repo = repo.scm_instance
 
         branches_group = ([(k, k) for k, v in
                            repo.branches.iteritems()], _("Branches"))
         hist_l.append(branches_group)
         choices.extend([x[0] for x in branches_group[0]])
 
         if repo.alias == 'hg':
             bookmarks_group = ([(k, k) for k, v in
                                 repo.bookmarks.iteritems()], _("Bookmarks"))
             hist_l.append(bookmarks_group)
             choices.extend([x[0] for x in bookmarks_group[0]])
 
         tags_group = ([(k, k) for k, v in
                        repo.tags.iteritems()], _("Tags"))
         hist_l.append(tags_group)
         choices.extend([x[0] for x in tags_group[0]])
 
         return choices, hist_l
 
     def install_git_hook(self, repo, force_create=False):
         """
         Creates a rhodecode hook inside a git repository
 
         :param repo: Instance of VCS repo
         :param force_create: Create even if same name hook exists
         """
 
         loc = jn(repo.path, 'hooks')
         if not repo.bare:
             loc = jn(repo.path, '.git', 'hooks')
         if not os.path.isdir(loc):
             os.makedirs(loc)
 
         tmpl_post = pkg_resources.resource_string(
             'rhodecode', jn('config', 'post_receive_tmpl.py')
         )
         tmpl_pre = pkg_resources.resource_string(
             'rhodecode', jn('config', 'pre_receive_tmpl.py')
         )
 
         for h_type, tmpl in [('pre', tmpl_pre), ('post', tmpl_post)]:
             _hook_file = jn(loc, '%s-receive' % h_type)
             _rhodecode_hook = False
             log.debug('Installing git hook in repo %s' % repo)
             if os.path.exists(_hook_file):
                 # let's take a look at this hook, maybe it's rhodecode ?
                 log.debug('hook exists, checking if it is from rhodecode')
                 _HOOK_VER_PAT = re.compile(r'^RC_HOOK_VER')
                 with open(_hook_file, 'rb') as f:
                     data = f.read()
                     matches = re.compile(r'(?:%s)\s*=\s*(.*)'
                                          % 'RC_HOOK_VER').search(data)
                     if matches:
                         try:
                             ver = matches.groups()[0]
                             log.debug('got %s it is rhodecode' % (ver))
                             _rhodecode_hook = True
                         except Exception:
                             log.error(traceback.format_exc())
             else:
                 # there is no hook in this dir, so we want to create one
                 _rhodecode_hook = True
 
             if _rhodecode_hook or force_create:
                 log.debug('writing %s hook file !' % h_type)
                 with open(_hook_file, 'wb') as f:
                     tmpl = tmpl.replace('_TMPL_', rhodecode.__version__)
                     f.write(tmpl)
                 os.chmod(_hook_file, 0755)
             else:
                 log.debug('skipping writing hook file')

rhodecode/model/validators.py

@@ -1,809 +1,809 b''
 """
 Set of generic validators
 """
 import os
 import re
 import formencode
 import logging
 from collections import defaultdict
 from pylons.i18n.translation import _
 from webhelpers.pylonslib.secure_form import authentication_token
 
 from formencode.validators import (
     UnicodeString, OneOf, Int, Number, Regex, Email, Bool, StringBoolean, Set,
-    NotEmpty, IPAddress, CIDR
+    NotEmpty, IPAddress, CIDR, String, FancyValidator
 )
 from rhodecode.lib.compat import OrderedSet
 from rhodecode.lib import ipaddr
 from rhodecode.lib.utils import repo_name_slug
 from rhodecode.lib.utils2 import safe_int, str2bool
 from rhodecode.model.db import RepoGroup, Repository, UserGroup, User,\
     ChangesetStatus
 from rhodecode.lib.exceptions import LdapImportError
 from rhodecode.config.routing import ADMIN_PREFIX
 from rhodecode.lib.auth import HasReposGroupPermissionAny, HasPermissionAny
 
 # silence warnings and pylint
 UnicodeString, OneOf, Int, Number, Regex, Email, Bool, StringBoolean, Set, \
-    NotEmpty, IPAddress, CIDR
+    NotEmpty, IPAddress, CIDR, String, FancyValidator
 
 log = logging.getLogger(__name__)
 
 
 class UniqueList(formencode.FancyValidator):
     """
     Unique List !
     """
     messages = dict(
         empty=_('Value cannot be an empty list'),
         missing_value=_('Value cannot be an empty list'),
     )
 
     def _to_python(self, value, state):
         if isinstance(value, list):
             return value
         elif isinstance(value, set):
             return list(value)
         elif isinstance(value, tuple):
             return list(value)
         elif value is None:
             return []
         else:
             return [value]
 
     def empty_value(self, value):
         return []
 
 
 class StateObj(object):
     """
     this is needed to translate the messages using _() in validators
     """
     _ = staticmethod(_)
 
 
 def M(self, key, state=None, **kwargs):
     """
     returns string from self.message based on given key,
     passed kw params are used to substitute %(named)s params inside
     translated strings
 
     :param msg:
     :param state:
     """
     if state is None:
         state = StateObj()
     else:
         state._ = staticmethod(_)
     #inject validator into state object
     return self.message(key, state, **kwargs)
 
 
 def ValidUsername(edit=False, old_data={}):
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'username_exists': _(u'Username "%(username)s" already exists'),
             'system_invalid_username':
                 _(u'Username "%(username)s" is forbidden'),
             'invalid_username':
                 _(u'Username may only contain alphanumeric characters '
                   'underscores, periods or dashes and must begin with '
                   'alphanumeric character')
         }
 
         def validate_python(self, value, state):
             if value in ['default', 'new_user']:
                 msg = M(self, 'system_invalid_username', state, username=value)
                 raise formencode.Invalid(msg, value, state)
             #check if user is unique
             old_un = None
             if edit:
                 old_un = User.get(old_data.get('user_id')).username
 
             if old_un != value or not edit:
                 if User.get_by_username(value, case_insensitive=True):
                     msg = M(self, 'username_exists', state, username=value)
                     raise formencode.Invalid(msg, value, state)
 
             if re.match(r'^[a-zA-Z0-9]{1}[a-zA-Z0-9\-\_\.]*$', value) is None:
                 msg = M(self, 'invalid_username', state)
                 raise formencode.Invalid(msg, value, state)
     return _validator
 
 
 def ValidRepoUser():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_username': _(u'Username %(username)s is not valid')
         }
 
         def validate_python(self, value, state):
             try:
                 User.query().filter(User.active == True)\
                     .filter(User.username == value).one()
             except Exception:
                 msg = M(self, 'invalid_username', state, username=value)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(username=msg)
                 )
 
     return _validator
 
 
 def ValidUserGroup(edit=False, old_data={}):
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_group': _(u'Invalid user group name'),
             'group_exist': _(u'User group "%(usergroup)s" already exists'),
             'invalid_usergroup_name':
                 _(u'user group name may only contain alphanumeric '
                   'characters underscores, periods or dashes and must begin '
                   'with alphanumeric character')
         }
 
         def validate_python(self, value, state):
             if value in ['default']:
                 msg = M(self, 'invalid_group', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(users_group_name=msg)
                 )
             #check if group is unique
             old_ugname = None
             if edit:
                 old_id = old_data.get('users_group_id')
                 old_ugname = UserGroup.get(old_id).users_group_name
 
             if old_ugname != value or not edit:
                 is_existing_group = UserGroup.get_by_group_name(value,
                                                         case_insensitive=True)
                 if is_existing_group:
                     msg = M(self, 'group_exist', state, usergroup=value)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(users_group_name=msg)
                     )
 
             if re.match(r'^[a-zA-Z0-9]{1}[a-zA-Z0-9\-\_\.]+$', value) is None:
                 msg = M(self, 'invalid_usergroup_name', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(users_group_name=msg)
                 )
 
     return _validator
 
 
 def ValidReposGroup(edit=False, old_data={}):
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'group_parent_id': _(u'Cannot assign this group as parent'),
             'group_exists': _(u'Group "%(group_name)s" already exists'),
             'repo_exists':
                 _(u'Repository with name "%(group_name)s" already exists')
         }
 
         def validate_python(self, value, state):
             # TODO WRITE VALIDATIONS
             group_name = value.get('group_name')
             group_parent_id = value.get('group_parent_id')
 
             # slugify repo group just in case :)
             slug = repo_name_slug(group_name)
 
             # check for parent of self
             parent_of_self = lambda: (
                 old_data['group_id'] == int(group_parent_id)
                 if group_parent_id else False
             )
             if edit and parent_of_self():
                 msg = M(self, 'group_parent_id', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(group_parent_id=msg)
                 )
 
             old_gname = None
             if edit:
                 old_gname = RepoGroup.get(old_data.get('group_id')).group_name
 
             if old_gname != group_name or not edit:
 
                 # check group
                 gr = RepoGroup.query()\
                       .filter(RepoGroup.group_name == slug)\
                       .filter(RepoGroup.group_parent_id == group_parent_id)\
                       .scalar()
 
                 if gr:
                     msg = M(self, 'group_exists', state, group_name=slug)
                     raise formencode.Invalid(msg, value, state,
                             error_dict=dict(group_name=msg)
                     )
 
                 # check for same repo
                 repo = Repository.query()\
                       .filter(Repository.repo_name == slug)\
                       .scalar()
 
                 if repo:
                     msg = M(self, 'repo_exists', state, group_name=slug)
                     raise formencode.Invalid(msg, value, state,
                             error_dict=dict(group_name=msg)
                     )
 
     return _validator
 
 
 def ValidPassword():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_password':
                 _(u'Invalid characters (non-ascii) in password')
         }
 
         def validate_python(self, value, state):
             try:
                 (value or '').decode('ascii')
             except UnicodeError:
                 msg = M(self, 'invalid_password', state)
                 raise formencode.Invalid(msg, value, state,)
     return _validator
 
 
 def ValidPasswordsMatch():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'password_mismatch': _(u'Passwords do not match'),
         }
 
         def validate_python(self, value, state):
 
             pass_val = value.get('password') or value.get('new_password')
             if pass_val != value['password_confirmation']:
                 msg = M(self, 'password_mismatch', state)
                 raise formencode.Invalid(msg, value, state,
                      error_dict=dict(password_confirmation=msg)
                 )
     return _validator
 
 
 def ValidAuth():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_password': _(u'invalid password'),
             'invalid_username': _(u'invalid user name'),
             'disabled_account': _(u'Your account is disabled')
         }
 
         def validate_python(self, value, state):
             from rhodecode.lib.auth import authenticate
 
             password = value['password']
             username = value['username']
 
             if not authenticate(username, password):
                 user = User.get_by_username(username)
                 if user and not user.active:
                     log.warning('user %s is disabled' % username)
                     msg = M(self, 'disabled_account', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(username=msg)
                     )
                 else:
                     log.warning('user %s failed to authenticate' % username)
                     msg = M(self, 'invalid_username', state)
                     msg2 = M(self, 'invalid_password', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(username=msg, password=msg2)
                     )
     return _validator
 
 
 def ValidAuthToken():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_token': _(u'Token mismatch')
         }
 
         def validate_python(self, value, state):
             if value != authentication_token():
                 msg = M(self, 'invalid_token', state)
                 raise formencode.Invalid(msg, value, state)
     return _validator
 
 
 def ValidRepoName(edit=False, old_data={}):
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_repo_name':
                 _(u'Repository name %(repo)s is disallowed'),
             'repository_exists':
                 _(u'Repository named %(repo)s already exists'),
             'repository_in_group_exists': _(u'Repository "%(repo)s" already '
                                             'exists in group "%(group)s"'),
             'same_group_exists': _(u'Repository group with name "%(repo)s" '
                                    'already exists')
         }
 
         def _to_python(self, value, state):
             repo_name = repo_name_slug(value.get('repo_name', ''))
             repo_group = value.get('repo_group')
             if repo_group:
                 gr = RepoGroup.get(repo_group)
                 group_path = gr.full_path
                 group_name = gr.group_name
                 # value needs to be aware of group name in order to check
                 # db key This is an actual just the name to store in the
                 # database
                 repo_name_full = group_path + RepoGroup.url_sep() + repo_name
             else:
                 group_name = group_path = ''
                 repo_name_full = repo_name
 
             value['repo_name'] = repo_name
             value['repo_name_full'] = repo_name_full
             value['group_path'] = group_path
             value['group_name'] = group_name
             return value
 
         def validate_python(self, value, state):
 
             repo_name = value.get('repo_name')
             repo_name_full = value.get('repo_name_full')
             group_path = value.get('group_path')
             group_name = value.get('group_name')
 
             if repo_name in [ADMIN_PREFIX, '']:
                 msg = M(self, 'invalid_repo_name', state, repo=repo_name)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(repo_name=msg)
                 )
 
             rename = old_data.get('repo_name') != repo_name_full
             create = not edit
             if rename or create:
 
                 if group_path != '':
                     if Repository.get_by_repo_name(repo_name_full):
                         msg = M(self, 'repository_in_group_exists', state,
                                 repo=repo_name, group=group_name)
                         raise formencode.Invalid(msg, value, state,
                             error_dict=dict(repo_name=msg)
                         )
                 elif RepoGroup.get_by_group_name(repo_name_full):
                         msg = M(self, 'same_group_exists', state,
                                 repo=repo_name)
                         raise formencode.Invalid(msg, value, state,
                             error_dict=dict(repo_name=msg)
                         )
 
                 elif Repository.get_by_repo_name(repo_name_full):
                         msg = M(self, 'repository_exists', state,
                                 repo=repo_name)
                         raise formencode.Invalid(msg, value, state,
                             error_dict=dict(repo_name=msg)
                         )
             return value
     return _validator
 
 
 def ValidForkName(*args, **kwargs):
     return ValidRepoName(*args, **kwargs)
 
 
 def SlugifyName():
     class _validator(formencode.validators.FancyValidator):
 
         def _to_python(self, value, state):
             return repo_name_slug(value)
 
         def validate_python(self, value, state):
             pass
 
     return _validator
 
 
 def ValidCloneUri():
     from rhodecode.lib.utils import make_ui
 
     def url_handler(repo_type, url, ui=None):
         if repo_type == 'hg':
             from rhodecode.lib.vcs.backends.hg.repository import MercurialRepository
             from mercurial.httppeer import httppeer
             if url.startswith('http'):
                 ## initially check if it's at least the proper URL
                 ## or does it pass basic auth
                 MercurialRepository._check_url(url)
                 httppeer(ui, url)._capabilities()
             elif url.startswith('svn+http'):
                 from hgsubversion.svnrepo import svnremoterepo
                 svnremoterepo(ui, url).capabilities
             elif url.startswith('git+http'):
                 raise NotImplementedError()
             else:
                 raise Exception('clone from URI %s not allowed' % (url))
 
         elif repo_type == 'git':
             from rhodecode.lib.vcs.backends.git.repository import GitRepository
             if url.startswith('http'):
                 ## initially check if it's at least the proper URL
                 ## or does it pass basic auth
                 GitRepository._check_url(url)
             elif url.startswith('svn+http'):
                 raise NotImplementedError()
             elif url.startswith('hg+http'):
                 raise NotImplementedError()
             else:
                 raise Exception('clone from URI %s not allowed' % (url))
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'clone_uri': _(u'invalid clone url'),
             'invalid_clone_uri': _(u'Invalid clone url, provide a '
                                     'valid clone http(s)/svn+http(s) url')
         }
 
         def validate_python(self, value, state):
             repo_type = value.get('repo_type')
             url = value.get('clone_uri')
 
             if not url:
                 pass
             else:
                 try:
                     url_handler(repo_type, url, make_ui('db', clear_session=False))
                 except Exception:
                     log.exception('Url validation failed')
                     msg = M(self, 'clone_uri')
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(clone_uri=msg)
                     )
     return _validator
 
 
 def ValidForkType(old_data={}):
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_fork_type': _(u'Fork have to be the same type as parent')
         }
 
         def validate_python(self, value, state):
             if old_data['repo_type'] != value:
                 msg = M(self, 'invalid_fork_type', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(repo_type=msg)
                 )
     return _validator
 
 
 def CanWriteGroup(old_data=None):
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'permission_denied': _(u"You don't have permissions "
                                    "to create repository in this group"),
             'permission_denied_root': _(u"no permission to create repository "
                                         "in root location")
         }
 
         def _to_python(self, value, state):
             #root location
             if value in [-1, "-1"]:
                 return None
             return value
 
         def validate_python(self, value, state):
             gr = RepoGroup.get(value)
             gr_name = gr.group_name if gr else None  # None means ROOT location
             val = HasReposGroupPermissionAny('group.write', 'group.admin')
             can_create_repos = HasPermissionAny('hg.admin', 'hg.create.repository')
             forbidden = not val(gr_name, 'can write into group validator')
             value_changed = True  # old_data['repo_group'].get('group_id') != safe_int(value)
             if value_changed:  # do check if we changed the value
                 #parent group need to be existing
                 if gr and forbidden:
                     msg = M(self, 'permission_denied', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(repo_type=msg)
                     )
                 ## check if we can write to root location !
                 elif gr is None and not can_create_repos():
                     msg = M(self, 'permission_denied_root', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(repo_type=msg)
                     )
 
     return _validator
 
 
 def CanCreateGroup(can_create_in_root=False):
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'permission_denied': _(u"You don't have permissions "
                                    "to create a group in this location")
         }
 
         def to_python(self, value, state):
             #root location
             if value in [-1, "-1"]:
                 return None
             return value
 
         def validate_python(self, value, state):
             gr = RepoGroup.get(value)
             gr_name = gr.group_name if gr else None  # None means ROOT location
 
             if can_create_in_root and gr is None:
                 #we can create in root, we're fine no validations required
                 return
 
             forbidden_in_root = gr is None and not can_create_in_root
             val = HasReposGroupPermissionAny('group.admin')
             forbidden = not val(gr_name, 'can create group validator')
             if forbidden_in_root or forbidden:
                 msg = M(self, 'permission_denied', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(group_parent_id=msg)
                 )
 
     return _validator
 
 
 def ValidPerms(type_='repo'):
     if type_ == 'repo_group':
         EMPTY_PERM = 'group.none'
     elif type_ == 'repo':
         EMPTY_PERM = 'repository.none'
     elif type_ == 'user_group':
         EMPTY_PERM = 'usergroup.none'
 
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'perm_new_member_name':
                 _(u'This username or user group name is not valid')
         }
 
         def to_python(self, value, state):
             perms_update = OrderedSet()
             perms_new = OrderedSet()
             # build a list of permission to update and new permission to create
 
             #CLEAN OUT ORG VALUE FROM NEW MEMBERS, and group them using
             new_perms_group = defaultdict(dict)
             for k, v in value.copy().iteritems():
                 if k.startswith('perm_new_member'):
                     del value[k]
                     _type, part = k.split('perm_new_member_')
                     args = part.split('_')
                     if len(args) == 1:
                         new_perms_group[args[0]]['perm'] = v
                     elif len(args) == 2:
                         _key, pos = args
                         new_perms_group[pos][_key] = v
 
             # fill new permissions in order of how they were added
             for k in sorted(map(int, new_perms_group.keys())):
                 perm_dict = new_perms_group[str(k)]
                 new_member = perm_dict.get('name')
                 new_perm = perm_dict.get('perm')
                 new_type = perm_dict.get('type')
                 if new_member and new_perm and new_type:
                     perms_new.add((new_member, new_perm, new_type))
 
             for k, v in value.iteritems():
                 if k.startswith('u_perm_') or k.startswith('g_perm_'):
                     member = k[7:]
                     t = {'u': 'user',
                          'g': 'users_group'
                     }[k[0]]
                     if member == 'default':
                         if str2bool(value.get('repo_private')):
                             # set none for default when updating to
                             # private repo protects agains form manipulation
                             v = EMPTY_PERM
                     perms_update.add((member, v, t))
 
             value['perms_updates'] = list(perms_update)
             value['perms_new'] = list(perms_new)
 
             # update permissions
             for k, v, t in perms_new:
                 try:
                     if t is 'user':
                         self.user_db = User.query()\
                             .filter(User.active == True)\
                             .filter(User.username == k).one()
                     if t is 'users_group':
                         self.user_db = UserGroup.query()\
                             .filter(UserGroup.users_group_active == True)\
                             .filter(UserGroup.users_group_name == k).one()
 
                 except Exception:
                     log.exception('Updated permission failed')
                     msg = M(self, 'perm_new_member_type', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(perm_new_member_name=msg)
                     )
             return value
     return _validator
 
 
 def ValidSettings():
     class _validator(formencode.validators.FancyValidator):
         def _to_python(self, value, state):
             # settings  form for users that are not admin
             # can't edit certain parameters, it's extra backup if they mangle
             # with forms
 
             forbidden_params = [
                 'user', 'repo_type', 'repo_enable_locking',
                 'repo_enable_downloads', 'repo_enable_statistics'
             ]
 
             for param in forbidden_params:
                 if param in value:
                     del value[param]
             return value
 
         def validate_python(self, value, state):
             pass
     return _validator
 
 
 def ValidPath():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_path': _(u'This is not a valid path')
         }
 
         def validate_python(self, value, state):
             if not os.path.isdir(value):
                 msg = M(self, 'invalid_path', state)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(paths_root_path=msg)
                 )
     return _validator
 
 
 def UniqSystemEmail(old_data={}):
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'email_taken': _(u'This e-mail address is already taken')
         }
 
         def _to_python(self, value, state):
             return value.lower()
 
         def validate_python(self, value, state):
             if (old_data.get('email') or '').lower() != value:
                 user = User.get_by_email(value, case_insensitive=True)
                 if user:
                     msg = M(self, 'email_taken', state)
                     raise formencode.Invalid(msg, value, state,
                         error_dict=dict(email=msg)
                     )
     return _validator
 
 
 def ValidSystemEmail():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'non_existing_email': _(u'e-mail "%(email)s" does not exist.')
         }
 
         def _to_python(self, value, state):
             return value.lower()
 
         def validate_python(self, value, state):
             user = User.get_by_email(value, case_insensitive=True)
             if user is None:
                 msg = M(self, 'non_existing_email', state, email=value)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(email=msg)
                 )
 
     return _validator
 
 
 def LdapLibValidator():
     class _validator(formencode.validators.FancyValidator):
         messages = {
 
         }
 
         def validate_python(self, value, state):
             try:
                 import ldap
                 ldap  # pyflakes silence !
             except ImportError:
                 raise LdapImportError()
 
     return _validator
 
 
 def AttrLoginValidator():
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'invalid_cn':
                   _(u'The LDAP Login attribute of the CN must be specified - '
                     'this is the name of the attribute that is equivalent '
                     'to "username"')
         }
         messages['empty'] = messages['invalid_cn']
 
     return _validator
 
 
 def NotReviewedRevisions(repo_id):
     class _validator(formencode.validators.FancyValidator):
         messages = {
             'rev_already_reviewed':
                   _(u'Revisions %(revs)s are already part of pull request '
                     'or have set status')
         }
 
         def validate_python(self, value, state):
             # check revisions if they are not reviewed, or a part of another
             # pull request
             statuses = ChangesetStatus.query()\
                 .filter(ChangesetStatus.revision.in_(value))\
                 .filter(ChangesetStatus.repo_id == repo_id)\
                 .all()
 
             errors = []
             for cs in statuses:
                 if cs.pull_request_id:
                     errors.append(['pull_req', cs.revision[:12]])
                 elif cs.status:
                     errors.append(['status', cs.revision[:12]])
 
             if errors:
                 revs = ','.join([x[1] for x in errors])
                 msg = M(self, 'rev_already_reviewed', state, revs=revs)
                 raise formencode.Invalid(msg, value, state,
                     error_dict=dict(revisions=revs)
                 )
 
     return _validator
 
 
 def ValidIp():
     class _validator(CIDR):
         messages = dict(
             badFormat=_('Please enter a valid IPv4 or IpV6 address'),
             illegalBits=_('The network size (bits) must be within the range'
                 ' of 0-32 (not %(bits)r)'))
 
         def to_python(self, value, state):
             v = super(_validator, self).to_python(value, state)
             v = v.strip()
             net = ipaddr.IPNetwork(address=v)
             if isinstance(net, ipaddr.IPv4Network):
                 #if IPv4 doesn't end with a mask, add /32
                 if '/' not in value:
                     v += '/32'
             if isinstance(net, ipaddr.IPv6Network):
                 #if IPv6 doesn't end with a mask, add /128
                 if '/' not in value:
                     v += '/128'
             return v
 
         def validate_python(self, value, state):
             try:
                 addr = value.strip()
                 #this raises an ValueError if address is not IpV4 or IpV6
                 ipaddr.IPNetwork(address=addr)
             except ValueError:
                 raise formencode.Invalid(self.message('badFormat', state),
                                          value, state)
 
     return _validator
 
 
 def FieldKey():
     class _validator(formencode.validators.FancyValidator):
         messages = dict(
             badFormat=_('Key name can only consist of letters, '
                         'underscore, dash or numbers'),)
 
         def validate_python(self, value, state):
             if not re.match('[a-zA-Z0-9_-]+$', value):
                 raise formencode.Invalid(self.message('badFormat', state),
                                          value, state)
     return _validator

rhodecode/public/css/contextbar.css

@@ -1,342 +1,348 b''
 /**
  * Stylesheets for the context bar
  */
 
 #quick .repo_switcher { background-image: url("../images/icons/database.png"); }
 #quick .journal { background-image: url("../images/icons/book.png"); }
+#quick .gists { background-image: url("../images/icons/note.png"); }
+#quick .gists-private { background-image: url("../images/icons/note_error.png"); }
+#quick .gists-new { background-image: url("../images/icons/note_add.png"); }
 #quick .search { background-image: url("../images/icons/search_16.png"); }
 #quick .admin { background-image: url("../images/icons/cog_edit.png"); }
 
 #context-bar a.follow { background-image: url("../images/icons/heart.png"); }
 #context-bar a.following { background-image: url("../images/icons/heart_delete.png"); }
 #context-bar a.fork { background-image: url("../images/icons/arrow_divide.png"); }
 #context-bar a.summary { background-image: url("../images/icons/clipboard_16.png"); }
 #context-bar a.changelogs { background-image: url("../images/icons/time.png"); }
 #context-bar a.files { background-image: url("../images/icons/file.png"); }
 #context-bar a.switch-to { background-image: url("../images/icons/arrow_switch.png"); }
 #context-bar a.options { background-image: url("../images/icons/table_gear.png"); }
 #context-bar a.forks { background-image: url("../images/icons/arrow_divide.png"); }
 #context-bar a.pull-request { background-image: url("../images/icons/arrow_join.png"); }
 #context-bar a.branches { background-image: url("../images/icons/arrow_branch.png"); }
 #context-bar a.tags { background-image: url("../images/icons/tag_blue.png"); }
 #context-bar a.bookmarks { background-image: url("../images/icons/tag_green.png"); }
 #context-bar a.settings { background-image: url("../images/icons/cog.png"); }
 #context-bar a.search { background-image: url("../images/icons/search_16.png"); }
 #context-bar a.admin { background-image: url("../images/icons/cog_edit.png"); }
 
 #context-bar a.journal { background-image: url("../images/icons/book.png"); }
+#context-bar a.gists { background-image: url("../images/icons/note.png"); }
+#context-bar a.gists-private { background-image: url("../images/icons/note_error.png"); }
+#context-bar a.gists-new { background-image: url("../images/icons/note_add.png"); }
 #context-bar a.repos { background-image: url("../images/icons/database_edit.png"); }
 #context-bar a.repos_groups { background-image: url("../images/icons/database_link.png"); }
 #context-bar a.users { background-image: url("../images/icons/user_edit.png"); }
 #context-bar a.groups { background-image: url("../images/icons/group_edit.png"); }
 #context-bar a.permissions { background-image: url("../images/icons/key.png"); }
 #context-bar a.ldap { background-image: url("../images/icons/server_key.png"); }
 #context-bar a.defaults { background-image: url("../images/icons/wrench.png"); }
 #context-bar a.settings { background-image: url("../images/icons/cog_edit.png"); }
 #context-bar a.compare_request { background-image: url('../images/icons/arrow_inout.png')}
 #context-bar a.locking_del { background-image: url('../images/icons/lock_delete.png')}
 #context-bar a.locking_add { background-image: url('../images/icons/lock_add.png')}
 
 #content #context-bar {
     position: relative;
     overflow: visible;
     background-color: #336699;
     border-top: 1px solid #517da8;
     border-bottom: 1px solid #003162;
     padding: 0 5px;
     min-height: 36px;
 }
 
 #header #header-inner #quick a,
 #content #context-bar,
 #content #context-bar a {
     color: #FFFFFF;
 }
 
 #header #header-inner #quick a:hover,
 #content #context-bar a:hover {
     text-decoration: none;
 }
 
 #content #context-bar .icon {
     display: inline-block;
     width: 16px;
     height: 16px;
     vertical-align: text-bottom;
 }
 
 ul.horizontal-list {
     display: block;
 }
 
 ul.horizontal-list > li {
     float: left;
     position: relative;
 }
 
 #header #header-inner #quick ul,
 ul.horizontal-list > li ul {
     position: absolute;
     display: none;
     right: 0;
     z-index: 999;
 }
 
 #header #header-inner #quick li:hover > ul,
 ul.horizontal-list li:hover > ul {
     display: block;
 }
 
 #header #header-inner #quick li ul li,
 ul.horizontal-list ul li {
     position: relative;
     border-bottom: 1px solid rgba(0,0,0,0.1);
     border-top: 1px solid rgba(255,255,255,0.1);
 }
 
 ul.horizontal-list > li ul ul {
     position: absolute;
     right: 100%;
     top: -1px;
     min-width: 200px;
     max-height: 400px;
     overflow-x: hidden;
     overflow-y: auto;
 }
 
 #header #header-inner #quick ul a,
 ul.horizontal-list li a {
     white-space: nowrap;
 }
 
 #breadcrumbs {
     float: left;
     padding: 6px 0 5px 0;
     padding-left: 5px;
     font-weight: bold;
     font-size: 14px;
 }
 
 #breadcrumbs span {
     font-weight: bold;
     font-size: 1.4em;
 }
 
 #header #header-inner #quick ul,
 #revision-changer,
 #context-pages,
 #context-pages ul {
     background: #3b6998; /* Old browsers */
     background: -moz-linear-gradient(top, #4574a2 0%, #2f5d8b 100%); /* FF3.6+ */
     background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#4574a2), color-stop(100%,#2f5d8b)); /* Chrome,Safari4+ */
     background: -webkit-linear-gradient(top, #4574a2 0%, #2f5d8b 100%); /* Chrome10+,Safari5.1+ */
     background: -o-linear-gradient(top, #4574a2 0%, #2f5d8b 100%); /* Opera 11.10+ */
     background: -ms-linear-gradient(top, #4574a2 0%, #2f5d8b 100%); /* IE10+ */
     background: linear-gradient(to bottom, #4574a2 0%, #2f5d8b 100%); /* W3C */
     /*Filter on IE will also use overflow:hidden implicitly, and that would clip our inner menus.*/
     /*filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#4574a2', endColorstr='#2f5d8b',GradientType=0 ); /* IE6-9 */*/
 }
 
 #header #header-inner #quick a,
 #context-actions a,
 #context-pages a {
     background-repeat: no-repeat;
     background-position: 10px 50%;
     padding-left: 30px;
 }
 
 #quick a,
 #context-pages ul ul a {
     padding-left: 10px;
 }
 
 ul#context-actions {
     display: inline-block;
     float: right;
     border-radius: 4px;
     background-image: linear-gradient(top, #4574a2 0%, #2f5d8b 100%);
 }
 
 #content ul#context-actions li {
     padding: 0px;
     border-right: 1px solid rgba(0,0,0,0.1);
     border-left: 1px solid rgba(255,255,255,0.1);
 }
 
 #context-actions a {
     display: block;
     cursor: pointer;
     background: none;
     border: none;
     margin: 0px;
     height: auto;
     padding: 10px 10px 10px 30px;
     background-repeat: no-repeat;
     background-position: 10px 50%;
     font-size: 1em;
 }
 
 #context-actions a {
     padding: 11px 10px 12px 30px;
 }
 
 #header #header-inner #quick li:hover,
 #revision-changer:hover,
 #context-pages li:hover,
 #context-actions li:hover,
 #content #context-actions li:hover,
 #header #header-inner #quick li.current,
 #context-pages li.current {
     background: #6388ad; /* Old browsers */
     background: -moz-linear-gradient(top, rgba(255,255,255,0.2) 0%, rgba(255,255,255,0.1) 100%); /* FF3.6+ */
     background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,rgba(255,255,255,0.1)), color-stop(100%,rgba(255,255,255,0))); /* Chrome,Safari4+ */
     background: -webkit-linear-gradient(top, rgba(255,255,255,0.2) 0%, rgba(255,255,255,0.1) 100%); /* Chrome10+,Safari5.1+ */
     background: -o-linear-gradient(top, rgba(255,255,255,0.2) 0%, rgba(255,255,255,0.1) 100%); /* Opera 11.10+ */
     background: -ms-linear-gradient(top, rgba(255,255,255,0.2) 0%, rgba(255,255,255,0.1) 100%); /* IE10+ */
     background: linear-gradient(to bottom, rgba(255,255,255,0.2) 0%, rgba(255,255,255,0.1) 100%); /* W3C */
     /*filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#88bfe8', endColorstr='#70b0e0',GradientType=0 ); /* IE6-9 */*/
 }
 
 
 #content #context-actions li:first-child {
     border-left: none;
     border-radius: 4px 0 0px 4px;
 }
 
 #content #context-actions li:last-child {
     border-right: none;
     border-radius: 0 4px 4px 0;
 }
 
 #content #context-actions .icon {
     margin: auto;
     margin-bottom: 5px;
     display: block;
     clear: both;
     float: none;
 }
 
 #content #context-pages .follow .show-following,
 #content #context-pages .following .show-follow {
     display: none;
 }
 
 #context-pages {
     float: right;
     border-left: 1px solid rgba(0,0,0,0.1);
 }
 
 #context-pages li.current {
     background: #535353; /* Old browsers */
     background: -moz-linear-gradient(top, #5d5d5d 0%, #484848 100%); /* FF3.6+ */
     background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#5d5d5d), color-stop(100%,#484848)); /* Chrome,Safari4+ */
     background: -webkit-linear-gradient(top, #5d5d5d 0%, #484848 100%); /* Chrome10+,Safari5.1+ */
     background: -o-linear-gradient(top, #5d5d5d 0%, #484848 100%); /* Opera 11.10+ */
     background: -ms-linear-gradient(top, #5d5d5d 0%, #484848 100%); /* IE10+ */
     background: linear-gradient(to bottom, #5d5d5d 0%, #484848 100%); /* W3C */
 }
 
 #content #context-pages .icon {
     margin-right: 5px;
 }
 
 #header #header-inner #quick li,
 #content #context-pages li {
     border-right: 1px solid rgba(0,0,0,0.1);
     border-left: 1px solid rgba(255,255,255,0.1);
     padding: 0;
 }
 
 #header #header-inner #quick li:last-child,
 #content #context-pages li:last-child {
     border-right: none;
 }
 
 #header #header-inner #quick > li:first-child {
     border-left: none;
 }
 
 #header #header-inner #quick > li:first-child > a {
     border-radius: 4px 0 0 4px;
 }
 
 #header #header-inner #quick a,
 #context-pages a,
 #context-pages .admin_menu a {
     display: block;
     padding: 0px 10px 1px 30px;
     padding-left: 30px;
     line-height: 35px;
 }
 
 #header #header-inner #quick a.thin,
 #context-pages a.thin,
 #context-pages .admin_menu a.thin {
     line-height: 28px !important;
 }
 
 #header #header-inner #quick a#quick_login_link {
     padding-left: 0px;
 }
 
 #header #header-inner #quick a {
     overflow: hidden;
 }
 #quick a.childs:after,
 #revision-changer:before,
 #context-pages a.childs:after,
 #context-pages a.dropdown:after {
     content: ' \25BE';
 }
 #context-pages a.childs {
     padding-right: 30px;
 }
 #context-pages a.childs:after {
     position: absolute;
     float: right;
     padding-left: 5px;
     padding-right: 5px;
 }
 
 #revision-changer:before {
     position: absolute;
     top: 0px;
     right: 0px;
     border-right: 1px solid rgba(0,0,0,0.1);
     height: 25px;
     padding-top: 10px;
     padding-right: 10px;
 }
 
 #context-pages li:last-child a {
     padding-right: 10px;
 }
 
 #context-bar #revision-changer {
     position: relative;
     cursor: pointer;
     border: none;
     padding: 0;
     margin: 0;
     color: #FFFFFF;
     font-size: 0.85em;
     padding: 2px 15px;
     padding-bottom: 3px;
     padding-right: 30px;
     border-right: 1px solid rgba(255,255,255,0.1);
 }
 
 #revision-changer .branch-name,
 #revision-changer .revision {
     display: block;
     text-align: center;
     line-height: 1.5em;
 }
 
 #revision-changer .branch-name {
     font-weight: bold;
 }
 
 #revision-changer .revision {
     text-transform: uppercase;
 }

rhodecode/public/css/pygments.css

@@ -1,173 +1,177 b''
 div.codeblock {
     overflow: auto;
     padding: 0px;
     border: 1px solid #ccc;
     background: #f8f8f8;
     font-size: 100%;
     line-height: 100%;
     /* new */
     line-height: 125%;
     -webkit-border-radius: 4px;
     -moz-border-radius: 4px;
     border-radius: 4px;
 }
 div.codeblock .code-header {
     border-bottom: 1px solid #CCCCCC;
     background: #EEEEEE;
-    padding: 10px 0 10px 0;
+    padding: 10px 0 5px 0;
 }
 
 div.codeblock .code-header .stats {
     clear: both;
-    padding: 6px 8px 6px 10px;
+    padding: 2px 8px 2px 14px;
     border-bottom: 1px solid rgb(204, 204, 204);
     height: 23px;
     margin-bottom: 6px;
 }
 
 div.codeblock .code-header .stats .left {
     float: left;
 }
 div.codeblock .code-header .stats .left.img {
     margin-top: -2px;
 }
 div.codeblock .code-header .stats .left.item {
     float: left;
     padding: 0 9px 0 9px;
     border-right: 1px solid #ccc;
 }
 div.codeblock .code-header .stats .left.item pre {
 }
 div.codeblock .code-header .stats .left.item.last {
     border-right: none;
 }
 div.codeblock .code-header .stats .buttons {
     float: right;
     padding-right: 4px;
 }
 
 div.codeblock .code-header .author {
-    margin-left: 25px;
+    margin-left: 15px;
     font-weight: bold;
     height: 25px;
 }
 div.codeblock .code-header .author .user {
     padding-top: 3px;
 }
 div.codeblock .code-header .commit {
-    margin-left: 25px;
+    margin-left: 15px;
     font-weight: normal;
     white-space: pre;
 }
 
+.code-highlighttable,
 div.codeblock .code-body table {
     width: 0 !important;
     border: 0px !important;
 }
+
+.code-highlighttable,
 div.codeblock .code-body table td {
     border: 0px !important;
 }
+
 div.code-body {
     background-color: #FFFFFF;
 }
 
 div.codeblock .code-header .search-path {
     padding: 0px 0px 0px 10px;
 }
 
 div.search-code-body {
     background-color: #FFFFFF;
     padding: 5px 0px 5px 10px;
 }
 
 div.search-code-body pre .match {
     background-color: #FAFFA6;
 }
 div.search-code-body pre .break {
     background-color: #DDE7EF;
     width: 100%;
     color: #747474;
     display: block;
 }
 div.annotatediv {
     margin-left: 2px;
     margin-right: 4px;
 }
 .code-highlight {
     padding: 0px;
     margin-top: 5px;
     margin-bottom: 5px;
-    border-left: 2px solid #ccc;
+    border-left: 1px solid #ccc;
 }
 .code-highlight pre, .linenodiv pre {
-    padding: 5px;
+    padding: 5px 2px 0px 5px;
     margin: 0;
 }
 .code-highlight pre div:target {
     background-color: #FFFFBE !important;
 }
-
+.linenos { padding: 0px !important; border:0px !important;}
 .linenos a { text-decoration: none; }
 
-.code { display: block; }
+.code { display: block; border:0px !important; }
 .code-highlight .hll, .codehilite .hll { background-color: #ffffcc }
 .code-highlight .c, .codehilite .c { color: #408080; font-style: italic } /* Comment */
 .code-highlight .err, .codehilite .err { border: 1px solid #FF0000 } /* Error */
 .code-highlight .k, .codehilite .k { color: #008000; font-weight: bold } /* Keyword */
 .code-highlight .o, .codehilite .o { color: #666666 } /* Operator */
 .code-highlight .cm, .codehilite .cm { color: #408080; font-style: italic } /* Comment.Multiline */
 .code-highlight .cp, .codehilite .cp { color: #BC7A00 } /* Comment.Preproc */
 .code-highlight .c1, .codehilite .c1 { color: #408080; font-style: italic } /* Comment.Single */
 .code-highlight .cs, .codehilite .cs { color: #408080; font-style: italic } /* Comment.Special */
 .code-highlight .gd, .codehilite .gd { color: #A00000 } /* Generic.Deleted */
 .code-highlight .ge, .codehilite .ge { font-style: italic } /* Generic.Emph */
 .code-highlight .gr, .codehilite .gr { color: #FF0000 } /* Generic.Error */
 .code-highlight .gh, .codehilite .gh { color: #000080; font-weight: bold } /* Generic.Heading */
 .code-highlight .gi, .codehilite .gi { color: #00A000 } /* Generic.Inserted */
 .code-highlight .go, .codehilite .go { color: #808080 } /* Generic.Output */
 .code-highlight .gp, .codehilite .gp { color: #000080; font-weight: bold } /* Generic.Prompt */
 .code-highlight .gs, .codehilite .gs { font-weight: bold } /* Generic.Strong */
 .code-highlight .gu, .codehilite .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
 .code-highlight .gt, .codehilite .gt { color: #0040D0 } /* Generic.Traceback */
 .code-highlight .kc, .codehilite .kc { color: #008000; font-weight: bold } /* Keyword.Constant */
 .code-highlight .kd, .codehilite .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */
 .code-highlight .kn, .codehilite .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */
 .code-highlight .kp, .codehilite .kp { color: #008000 } /* Keyword.Pseudo */
 .code-highlight .kr, .codehilite .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */
 .code-highlight .kt, .codehilite .kt { color: #B00040 } /* Keyword.Type */
 .code-highlight .m, .codehilite .m { color: #666666 } /* Literal.Number */
 .code-highlight .s, .codehilite .s { color: #BA2121 } /* Literal.String */
 .code-highlight .na, .codehilite .na { color: #7D9029 } /* Name.Attribute */
 .code-highlight .nb, .codehilite .nb { color: #008000 } /* Name.Builtin */
 .code-highlight .nc, .codehilite .nc { color: #0000FF; font-weight: bold } /* Name.Class */
 .code-highlight .no, .codehilite .no { color: #880000 } /* Name.Constant */
 .code-highlight .nd, .codehilite .nd { color: #AA22FF } /* Name.Decorator */
 .code-highlight .ni, .codehilite .ni { color: #999999; font-weight: bold } /* Name.Entity */
 .code-highlight .ne, .codehilite .ne { color: #D2413A; font-weight: bold } /* Name.Exception */
 .code-highlight .nf, .codehilite .nf { color: #0000FF } /* Name.Function */
 .code-highlight .nl, .codehilite .nl { color: #A0A000 } /* Name.Label */
 .code-highlight .nn, .codehilite .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */
 .code-highlight .nt, .codehilite .nt { color: #008000; font-weight: bold } /* Name.Tag */
 .code-highlight .nv, .codehilite .nv { color: #19177C } /* Name.Variable */
 .code-highlight .ow, .codehilite .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */
 .code-highlight .w, .codehilite .w { color: #bbbbbb } /* Text.Whitespace */
 .code-highlight .mf, .codehilite .mf { color: #666666 } /* Literal.Number.Float */
 .code-highlight .mh, .codehilite .mh { color: #666666 } /* Literal.Number.Hex */
 .code-highlight .mi, .codehilite .mi { color: #666666 } /* Literal.Number.Integer */
 .code-highlight .mo, .codehilite .mo { color: #666666 } /* Literal.Number.Oct */
 .code-highlight .sb, .codehilite .sb { color: #BA2121 } /* Literal.String.Backtick */
 .code-highlight .sc, .codehilite .sc { color: #BA2121 } /* Literal.String.Char */
 .code-highlight .sd, .codehilite .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */
 .code-highlight .s2, .codehilite .s2 { color: #BA2121 } /* Literal.String.Double */
 .code-highlight .se, .codehilite .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */
 .code-highlight .sh, .codehilite .sh { color: #BA2121 } /* Literal.String.Heredoc */
 .code-highlight .si, .codehilite .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */
 .code-highlight .sx, .codehilite .sx { color: #008000 } /* Literal.String.Other */
 .code-highlight .sr, .codehilite .sr { color: #BB6688 } /* Literal.String.Regex */
 .code-highlight .s1, .codehilite .s1 { color: #BA2121 } /* Literal.String.Single */
 .code-highlight .ss, .codehilite .ss { color: #19177C } /* Literal.String.Symbol */
 .code-highlight .bp, .codehilite .bp { color: #008000 } /* Name.Builtin.Pseudo */
 .code-highlight .vc, .codehilite .vc { color: #19177C } /* Name.Variable.Class */
 .code-highlight .vg, .codehilite .vg { color: #19177C } /* Name.Variable.Global */
 .code-highlight .vi, .codehilite .vi { color: #19177C } /* Name.Variable.Instance */
 .code-highlight .il, .codehilite .il { color: #666666 } /* Literal.Number.Integer.Long */