Merge with upstream

rhodecode/lib/compat.py

@@ -0,0 +1,360 b''
+# -*- coding: utf-8 -*-
+"""
+    rhodecode.lib.compat
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Python backward compatibility functions and common libs
+    
+    
+    :created_on: Oct 7, 2011
+    :author: marcink
+    :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
+    :license: GPLv3, see COPYING for more details.
+"""
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#==============================================================================
+# json
+#==============================================================================
+try:
+    import json
+except ImportError:
+    import simplejson as json
+
+
+#==============================================================================
+# izip_longest
+#==============================================================================
+try:
+    from itertools import izip_longest
+except ImportError:
+    import itertools
+
+    def izip_longest(*args, **kwds): # noqa
+        fillvalue = kwds.get("fillvalue")
+
+        def sentinel(counter=([fillvalue] * (len(args) - 1)).pop):
+            yield counter() # yields the fillvalue, or raises IndexError
+
+        fillers = itertools.repeat(fillvalue)
+        iters = [itertools.chain(it, sentinel(), fillers)
+                    for it in args]
+        try:
+            for tup in itertools.izip(*iters):
+                yield tup
+        except IndexError:
+            pass
+
+
+#==============================================================================
+# OrderedDict
+#==============================================================================
+
+# Python Software Foundation License
+
+# XXX: it feels like using the class with "is" and "is not" instead of "==" and
+# "!=" should be faster.
+class _Nil(object):
+
+    def __repr__(self):
+        return "nil"
+
+    def __eq__(self, other):
+        if (isinstance(other, _Nil)):
+            return True
+        else:
+            return NotImplemented
+
+    def __ne__(self, other):
+        if (isinstance(other, _Nil)):
+            return False
+        else:
+            return NotImplemented
+
+_nil = _Nil()
+
+class _odict(object):
+    """Ordered dict data structure, with O(1) complexity for dict operations
+    that modify one element.
+    
+    Overwriting values doesn't change their original sequential order.
+    """
+
+    def _dict_impl(self):
+        return None
+
+    def __init__(self, data=(), **kwds):
+        """This doesn't accept keyword initialization as normal dicts to avoid
+        a trap - inside a function or method the keyword args are accessible
+        only as a dict, without a defined order, so their original order is
+        lost.
+        """
+        if kwds:
+            raise TypeError("__init__() of ordered dict takes no keyword "
+                            "arguments to avoid an ordering trap.")
+        self._dict_impl().__init__(self)
+        # If you give a normal dict, then the order of elements is undefined
+        if hasattr(data, "iteritems"):
+            for key, val in data.iteritems():
+                self[key] = val
+        else:
+            for key, val in data:
+                self[key] = val
+
+    # Double-linked list header
+    def _get_lh(self):
+        dict_impl = self._dict_impl()
+        if not hasattr(self, '_lh'):
+            dict_impl.__setattr__(self, '_lh', _nil)
+        return dict_impl.__getattribute__(self, '_lh')
+
+    def _set_lh(self, val):
+        self._dict_impl().__setattr__(self, '_lh', val)
+
+    lh = property(_get_lh, _set_lh)
+
+    # Double-linked list tail
+    def _get_lt(self):
+        dict_impl = self._dict_impl()
+        if not hasattr(self, '_lt'):
+            dict_impl.__setattr__(self, '_lt', _nil)
+        return dict_impl.__getattribute__(self, '_lt')
+
+    def _set_lt(self, val):
+        self._dict_impl().__setattr__(self, '_lt', val)
+
+    lt = property(_get_lt, _set_lt)
+
+    def __getitem__(self, key):
+        return self._dict_impl().__getitem__(self, key)[1]
+
+    def __setitem__(self, key, val):
+        dict_impl = self._dict_impl()
+        try:
+            dict_impl.__getitem__(self, key)[1] = val
+        except KeyError, e:
+            new = [dict_impl.__getattribute__(self, 'lt'), val, _nil]
+            dict_impl.__setitem__(self, key, new)
+            if dict_impl.__getattribute__(self, 'lt') == _nil:
+                dict_impl.__setattr__(self, 'lh', key)
+            else:
+                dict_impl.__getitem__(
+                    self, dict_impl.__getattribute__(self, 'lt'))[2] = key
+            dict_impl.__setattr__(self, 'lt', key)
+
+    def __delitem__(self, key):
+        dict_impl = self._dict_impl()
+        pred, _ , succ = self._dict_impl().__getitem__(self, key)
+        if pred == _nil:
+            dict_impl.__setattr__(self, 'lh', succ)
+        else:
+            dict_impl.__getitem__(self, pred)[2] = succ
+        if succ == _nil:
+            dict_impl.__setattr__(self, 'lt', pred)
+        else:
+            dict_impl.__getitem__(self, succ)[0] = pred
+        dict_impl.__delitem__(self, key)
+
+    def __contains__(self, key):
+        return key in self.keys()
+
+    def __len__(self):
+        return len(self.keys())
+
+    def __str__(self):
+        pairs = ("%r: %r" % (k, v) for k, v in self.iteritems())
+        return "{%s}" % ", ".join(pairs)
+
+    def __repr__(self):
+        if self:
+            pairs = ("(%r, %r)" % (k, v) for k, v in self.iteritems())
+            return "odict([%s])" % ", ".join(pairs)
+        else:
+            return "odict()"
+
+    def get(self, k, x=None):
+        if k in self:
+            return self._dict_impl().__getitem__(self, k)[1]
+        else:
+            return x
+
+    def __iter__(self):
+        dict_impl = self._dict_impl()
+        curr_key = dict_impl.__getattribute__(self, 'lh')
+        while curr_key != _nil:
+            yield curr_key
+            curr_key = dict_impl.__getitem__(self, curr_key)[2]
+
+    iterkeys = __iter__
+
+    def keys(self):
+        return list(self.iterkeys())
+
+    def itervalues(self):
+        dict_impl = self._dict_impl()
+        curr_key = dict_impl.__getattribute__(self, 'lh')
+        while curr_key != _nil:
+            _, val, curr_key = dict_impl.__getitem__(self, curr_key)
+            yield val
+
+    def values(self):
+        return list(self.itervalues())
+
+    def iteritems(self):
+        dict_impl = self._dict_impl()
+        curr_key = dict_impl.__getattribute__(self, 'lh')
+        while curr_key != _nil:
+            _, val, next_key = dict_impl.__getitem__(self, curr_key)
+            yield curr_key, val
+            curr_key = next_key
+
+    def items(self):
+        return list(self.iteritems())
+
+    def sort(self, cmp=None, key=None, reverse=False):
+        items = [(k, v) for k, v in self.items()]
+        if cmp is not None:
+            items = sorted(items, cmp=cmp)
+        elif key is not None:
+            items = sorted(items, key=key)
+        else:
+            items = sorted(items, key=lambda x: x[1])
+        if reverse:
+            items.reverse()
+        self.clear()
+        self.__init__(items)
+
+    def clear(self):
+        dict_impl = self._dict_impl()
+        dict_impl.clear(self)
+        dict_impl.__setattr__(self, 'lh', _nil)
+        dict_impl.__setattr__(self, 'lt', _nil)
+
+    def copy(self):
+        return self.__class__(self)
+
+    def update(self, data=(), **kwds):
+        if kwds:
+            raise TypeError("update() of ordered dict takes no keyword "
+                            "arguments to avoid an ordering trap.")
+        if hasattr(data, "iteritems"):
+            data = data.iteritems()
+        for key, val in data:
+            self[key] = val
+
+    def setdefault(self, k, x=None):
+        try:
+            return self[k]
+        except KeyError:
+            self[k] = x
+            return x
+
+    def pop(self, k, x=_nil):
+        try:
+            val = self[k]
+            del self[k]
+            return val
+        except KeyError:
+            if x == _nil:
+                raise
+            return x
+
+    def popitem(self):
+        try:
+            dict_impl = self._dict_impl()
+            key = dict_impl.__getattribute__(self, 'lt')
+            return key, self.pop(key)
+        except KeyError:
+            raise KeyError("'popitem(): ordered dictionary is empty'")
+
+    def riterkeys(self):
+        """To iterate on keys in reversed order.
+        """
+        dict_impl = self._dict_impl()
+        curr_key = dict_impl.__getattribute__(self, 'lt')
+        while curr_key != _nil:
+            yield curr_key
+            curr_key = dict_impl.__getitem__(self, curr_key)[0]
+
+    __reversed__ = riterkeys
+
+    def rkeys(self):
+        """List of the keys in reversed order.
+        """
+        return list(self.riterkeys())
+
+    def ritervalues(self):
+        """To iterate on values in reversed order.
+        """
+        dict_impl = self._dict_impl()
+        curr_key = dict_impl.__getattribute__(self, 'lt')
+        while curr_key != _nil:
+            curr_key, val, _ = dict_impl.__getitem__(self, curr_key)
+            yield val
+
+    def rvalues(self):
+        """List of the values in reversed order.
+        """
+        return list(self.ritervalues())
+
+    def riteritems(self):
+        """To iterate on (key, value) in reversed order.
+        """
+        dict_impl = self._dict_impl()
+        curr_key = dict_impl.__getattribute__(self, 'lt')
+        while curr_key != _nil:
+            pred_key, val, _ = dict_impl.__getitem__(self, curr_key)
+            yield curr_key, val
+            curr_key = pred_key
+
+    def ritems(self):
+        """List of the (key, value) in reversed order.
+        """
+        return list(self.riteritems())
+
+    def firstkey(self):
+        if self:
+            return self._dict_impl().__getattribute__(self, 'lh')
+        else:
+            raise KeyError("'firstkey(): ordered dictionary is empty'")
+
+    def lastkey(self):
+        if self:
+            return self._dict_impl().__getattribute__(self, 'lt')
+        else:
+            raise KeyError("'lastkey(): ordered dictionary is empty'")
+
+    def as_dict(self):
+        return self._dict_impl()(self.items())
+
+    def _repr(self):
+        """_repr(): low level repr of the whole data contained in the odict.
+        Useful for debugging.
+        """
+        dict_impl = self._dict_impl()
+        form = "odict low level repr lh,lt,data: %r, %r, %s"
+        return form % (dict_impl.__getattribute__(self, 'lh'),
+                       dict_impl.__getattribute__(self, 'lt'),
+                       dict_impl.__repr__(self))
+
+class OrderedDict(_odict, dict):
+
+    def _dict_impl(self):
+        return dict
+
+
+#==============================================================================
+# OrderedSet
+#==============================================================================
+from sqlalchemy.util import OrderedSet

README.rst

@@ -1,148 +1,149 b''
 =================================================
 Welcome to RhodeCode (RhodiumCode) documentation!
 =================================================
 
-``RhodeCode`` (formerly hg-app) is a Pylons framework based Mercurial repository 
+``RhodeCode`` is a Pylons framework based Mercurial repository 
 browser/management tool with a built in push/pull server and full text search.
 It works on http/https and has a built in permission/authentication system with 
-the ability to authenticate via LDAP.
+the ability to authenticate via LDAP or ActiveDirectory. RhodeCode also supports
+simple API so it's easy integrable with existing systems.
 
 RhodeCode is similar in some respects to github or bitbucket_, 
 however RhodeCode can be run as standalone hosted application on your own server.  
 It is open source and donation ware and focuses more on providing a customized, 
 self administered interface for Mercurial(and soon GIT) repositories. 
 RhodeCode is powered by a vcs_ library that Lukasz Balcerzak and I created to 
 handle multiple different version control systems.
 
 RhodeCode uses `Semantic Versioning <http://semver.org/>`_
 
 RhodeCode demo
 --------------
 
 http://demo.rhodecode.org
 
 The default access is anonymous but you can login to an administrative account
 using the following credentials:
 
 - username: demo
 - password: demo12
 
 Source code
 -----------
 
 The latest sources can be obtained from official RhodeCode instance
 https://secure.rhodecode.org 
 
 
 MIRRORS:
 
 Issue tracker and sources at bitbucket_
 
 http://bitbucket.org/marcinkuzminski/rhodecode
 
 Sources at github_
 
 https://github.com/marcinkuzminski/rhodecode
 
 Installation
 ------------
 
 Please visit http://packages.python.org/RhodeCode/installation.html
 
 
 RhodeCode Features
 ------------------
 
 - Has it's own middleware to handle mercurial_ protocol requests. 
   Each request can be logged and authenticated.
 - Runs on threads unlike hgweb. You can make multiple pulls/pushes simultaneous.
   Supports http/https and LDAP
 - Full permissions (private/read/write/admin) and authentication per project. 
   One account for web interface and mercurial_ push/pull/clone operations.
 - Have built in users groups for easier permission management
 - Repository groups let you group repos and manage them easier.
 - Users can fork other users repo. RhodeCode have also compare view to see
   combined changeset for all changeset made within single push.
 - Build in commit-api let's you add, edit and commit files right from RhodeCode
   interface using simple editor or upload form for binaries.
 - Mako templates let's you customize the look and feel of the application.
 - Beautiful diffs, annotations and source code browsing all colored by pygments. 
   Raw diffs are made in git-diff format, including git_ binary-patches
 - Mercurial_ branch graph and yui-flot powered graphs with zooming and statistics
 - Admin interface with user/permission management. Admin activity journal, logs
   pulls, pushes, forks, registrations and other actions made by all users.
 - Server side forks. It is possible to fork a project and modify it freely 
   without breaking the main repository. You can even write Your own hooks 
   and install them
 - Full text search powered by Whoosh on the source files, and file names.
   Build in indexing daemons, with optional incremental index build
   (no external search servers required all in one application)
 - Setup project descriptions and info inside built in db for easy, non 
   file-system operations
 - Intelligent cache with invalidation after push or project change, provides 
   high performance and always up to date data.
 - Rss / atom feeds, gravatar support, download sources as zip/tar/gz
 - Async tasks for speed and performance using celery_ (works without them too)  
 - Backup scripts can do backup of whole app and send it over scp to desired 
   location 
 - Based on pylons / sqlalchemy / sqlite / whoosh / vcs
 
 
 .. include:: ./docs/screenshots.rst
     
     
 Incoming / Plans
 ----------------
 
 - Finer granular permissions per branch, repo group or subrepo
 - pull requests and web based merges
 - notification and message system 
 - SSH based authentication with server side key management
 - Code review (probably based on hg-review)
 - Full git_ support, with push/pull server (currently in beta tests)
 - Redmine and other bugtrackers integration
 - Commit based built in wiki system
 - More statistics and graph (global annotation + some more statistics)
 - Other advancements as development continues (or you can of course make 
   additions and or requests)
 
 License
 -------
 
 ``RhodeCode`` is released under the GPLv3 license.
 
 
 Mailing group Q&A
 -----------------
 
 Join the `Google group <http://groups.google.com/group/rhodecode>`_
 
 Open an issue at `issue tracker <http://bitbucket.org/marcinkuzminski/rhodecode/issues>`_
 
 Join #rhodecode on FreeNode (irc.freenode.net)
 or use http://webchat.freenode.net/?channels=rhodecode for web access to irc.
 
 Online documentation
 --------------------
 
 Online documentation for the current version of RhodeCode is available at
 http://packages.python.org/RhodeCode/.
 You may also build the documentation for yourself - go into ``docs/`` and run::
 
    make html
 
 (You need to have sphinx_ installed to build the documentation. If you don't
 have sphinx_ installed you can install it via the command: 
 ``easy_install sphinx``)
  
 .. _virtualenv: http://pypi.python.org/pypi/virtualenv
 .. _python: http://www.python.org/
 .. _sphinx: http://sphinx.pocoo.org/
 .. _mercurial: http://mercurial.selenic.com/
 .. _bitbucket: http://bitbucket.org/
 .. _github: http://github.com/
 .. _subversion: http://subversion.tigris.org/
 .. _git: http://git-scm.com/
 .. _celery: http://celeryproject.org/
 .. _Sphinx: http://sphinx.pocoo.org/
 .. _vcs: http://pypi.python.org/pypi/vcs
  No newline at end of file

docs/changelog.rst

@@ -1,341 +1,354 b''
 .. _changelog:
 
 Changelog
 =========
 
-1.2.0 (**2011-XX-XX**)
+
+1.3.0 (**XXXX-XX-XX**)
 ======================
 
 :status: in-progress
 :branch: beta
 
 news
 ----
 
+fixes
+-----
+
+
+
+
+1.2.0 (**2011-10-07**)
+======================
+
+news
+----
+
 - implemented #47 repository groups
 - implemented #89 Can setup google analytics code from settings menu
 - implemented #91 added nicer looking archive urls with more download options
   like tags, branches
 - implemented #44 into file browsing, and added follow branch option
 - implemented #84 downloads can be enabled/disabled for each repository
 - anonymous repository can be cloned without having to pass default:default
   into clone url
 - fixed #90 whoosh indexer can index chooses repositories passed in command 
   line
 - extended journal with day aggregates and paging
 - implemented #107 source code lines highlight ranges
 - implemented #93 customizable changelog on combined revision ranges - 
   equivalent of githubs compare view 
 - implemented #108 extended and more powerful LDAP configuration
 - implemented #56 users groups
 - major code rewrites optimized codes for speed and memory usage
 - raw and diff downloads are now in git format
 - setup command checks for write access to given path
 - fixed many issues with international characters and unicode. It uses utf8
   decode with replace to provide less errors even with non utf8 encoded strings
 - #125 added API KEY access to feeds
 - #109 Repository can be created from external Mercurial link (aka. remote 
   repository, and manually updated (via pull) from admin panel
 - beta git support - push/pull server + basic view for git repos
 - added followers page and forks page
 - server side file creation (with binary file upload interface) 
   and edition with commits powered by codemirror 
 - #111 file browser file finder, quick lookup files on whole file tree 
 - added quick login sliding menu into main page
 - changelog uses lazy loading of affected files details, in some scenarios 
   this can improve speed of changelog page dramatically especially for 
   larger repositories.
 - implements #214 added support for downloading subrepos in download menu.
 - Added basic API for direct operations on rhodecode via JSON
 - Implemented advanced hook management
 
 fixes
 -----
 
 - fixed file browser bug, when switching into given form revision the url was 
   not changing
 - fixed propagation to error controller on simplehg and simplegit middlewares
 - fixed error when trying to make a download on empty repository
 - fixed problem with '[' chars in commit messages in journal
 - fixed #99 Unicode errors, on file node paths with non utf-8 characters
 - journal fork fixes
 - removed issue with space inside renamed repository after deletion
 - fixed strange issue on formencode imports
 - fixed #126 Deleting repository on Windows, rename used incompatible chars. 
 - #150 fixes for errors on repositories mapped in db but corrupted in 
   filesystem
 - fixed problem with ascendant characters in realm #181
 - fixed problem with sqlite file based database connection pool
 - whoosh indexer and code stats share the same dynamic extensions map
 - fixes #188 - relationship delete of repo_to_perm entry on user removal
 - fixes issue #189 Trending source files shows "show more" when no more exist
 - fixes issue #197 Relative paths for pidlocks
 - fixes issue #198 password will require only 3 chars now for login form
 - fixes issue #199 wrong redirection for non admin users after creating a repository
 - fixes issues #202, bad db constraint made impossible to attach same group 
   more than one time. Affects only mysql/postgres
 - fixes #218 os.kill patch for windows was missing sig param
 - improved rendering of dag (they are not trimmed anymore when number of 
   heads exceeds 5)
     
 1.1.8 (**2011-04-12**)
 ======================
 
 news
 ----
 
 - improved windows support
 
 fixes
 -----
 
 - fixed #140 freeze of python dateutil library, since new version is python2.x
   incompatible
 - setup-app will check for write permission in given path
 - cleaned up license info issue #149
 - fixes for issues #137,#116 and problems with unicode and accented characters.
 - fixes crashes on gravatar, when passed in email as unicode
 - fixed tooltip flickering problems
 - fixed came_from redirection on windows
 - fixed logging modules, and sql formatters
 - windows fixes for os.kill issue #133
 - fixes path splitting for windows issues #148
 - fixed issue #143 wrong import on migration to 1.1.X
 - fixed problems with displaying binary files, thanks to Thomas Waldmann
 - removed name from archive files since it's breaking ui for long repo names
 - fixed issue with archive headers sent to browser, thanks to Thomas Waldmann
 - fixed compatibility for 1024px displays, and larger dpi settings, thanks to 
   Thomas Waldmann
 - fixed issue #166 summary pager was skipping 10 revisions on second page
 
 
 1.1.7 (**2011-03-23**)
 ======================
 
 news
 ----
 
 fixes
 -----
 
 - fixed (again) #136 installation support for FreeBSD
 
 
 1.1.6 (**2011-03-21**)
 ======================
 
 news
 ----
 
 fixes
 -----
 
 - fixed #136 installation support for FreeBSD
 - RhodeCode will check for python version during installation
 
 1.1.5 (**2011-03-17**)
 ======================
 
 news
 ----
 
 - basic windows support, by exchanging pybcrypt into sha256 for windows only
   highly inspired by idea of mantis406
 
 fixes
 -----
 
 - fixed sorting by author in main page
 - fixed crashes with diffs on binary files
 - fixed #131 problem with boolean values for LDAP
 - fixed #122 mysql problems thanks to striker69 
 - fixed problem with errors on calling raw/raw_files/annotate functions 
   with unknown revisions
 - fixed returned rawfiles attachment names with international character
 - cleaned out docs, big thanks to Jason Harris
 
 1.1.4 (**2011-02-19**)
 ======================
 
 news
 ----
 
 fixes
 -----
 
 - fixed formencode import problem on settings page, that caused server crash
   when that page was accessed as first after server start
 - journal fixes
 - fixed option to access repository just by entering http://server/<repo_name> 
 
 1.1.3 (**2011-02-16**)
 ======================
 
 news
 ----
 
 - implemented #102 allowing the '.' character in username
 - added option to access repository just by entering http://server/<repo_name>
 - celery task ignores result for better performance
 
 fixes
 -----
 
 - fixed ehlo command and non auth mail servers on smtp_lib. Thanks to 
   apollo13 and Johan Walles
 - small fixes in journal
 - fixed problems with getting setting for celery from .ini files
 - registration, password reset and login boxes share the same title as main 
   application now
 - fixed #113: to high permissions to fork repository
 - fixed problem with '[' chars in commit messages in journal
 - removed issue with space inside renamed repository after deletion
 - db transaction fixes when filesystem repository creation failed
 - fixed #106 relation issues on databases different than sqlite
 - fixed static files paths links to use of url() method
 
 1.1.2 (**2011-01-12**)
 ======================
 
 news
 ----
 
 
 fixes
 -----
 
 - fixes #98 protection against float division of percentage stats
 - fixed graph bug
 - forced webhelpers version since it was making troubles during installation 
 
 1.1.1 (**2011-01-06**)
 ======================
  
 news
 ----
 
 - added force https option into ini files for easier https usage (no need to
   set server headers with this options)
 - small css updates
 
 fixes
 -----
 
 - fixed #96 redirect loop on files view on repositories without changesets
 - fixed #97 unicode string passed into server header in special cases (mod_wsgi)
   and server crashed with errors
 - fixed large tooltips problems on main page
 - fixed #92 whoosh indexer is more error proof
 
 1.1.0 (**2010-12-18**)
 ======================
 
 news
 ----
 
 - rewrite of internals for vcs >=0.1.10
 - uses mercurial 1.7 with dotencode disabled for maintaining compatibility 
   with older clients
 - anonymous access, authentication via ldap
 - performance upgrade for cached repos list - each repository has it's own 
   cache that's invalidated when needed.
 - performance upgrades on repositories with large amount of commits (20K+)
 - main page quick filter for filtering repositories
 - user dashboards with ability to follow chosen repositories actions
 - sends email to admin on new user registration
 - added cache/statistics reset options into repository settings
 - more detailed action logger (based on hooks) with pushed changesets lists
   and options to disable those hooks from admin panel
 - introduced new enhanced changelog for merges that shows more accurate results
 - new improved and faster code stats (based on pygments lexers mapping tables, 
   showing up to 10 trending sources for each repository. Additionally stats
   can be disabled in repository settings.
 - gui optimizations, fixed application width to 1024px
 - added cut off (for large files/changesets) limit into config files
 - whoosh, celeryd, upgrade moved to paster command
 - other than sqlite database backends can be used
 
 fixes
 -----
 
 - fixes #61 forked repo was showing only after cache expired
 - fixes #76 no confirmation on user deletes
 - fixes #66 Name field misspelled
 - fixes #72 block user removal when he owns repositories
 - fixes #69 added password confirmation fields
 - fixes #87 RhodeCode crashes occasionally on updating repository owner
 - fixes #82 broken annotations on files with more than 1 blank line at the end
 - a lot of fixes and tweaks for file browser
 - fixed detached session issues
 - fixed when user had no repos he would see all repos listed in my account
 - fixed ui() instance bug when global hgrc settings was loaded for server 
   instance and all hgrc options were merged with our db ui() object
 - numerous small bugfixes
  
 (special thanks for TkSoh for detailed feedback)
 
 
 1.0.2 (**2010-11-12**)
 ======================
 
 news
 ----
 
 - tested under python2.7
 - bumped sqlalchemy and celery versions
 
 fixes
 -----
 
 - fixed #59 missing graph.js
 - fixed repo_size crash when repository had broken symlinks
 - fixed python2.5 crashes.
 
 
 1.0.1 (**2010-11-10**)
 ======================
 
 news
 ----
 
 - small css updated
 
 fixes
 -----
 
 - fixed #53 python2.5 incompatible enumerate calls
 - fixed #52 disable mercurial extension for web
 - fixed #51 deleting repositories don't delete it's dependent objects
 
 
 1.0.0 (**2010-11-02**)
 ======================
 
 - security bugfix simplehg wasn't checking for permissions on commands
   other than pull or push.
 - fixed doubled messages after push or pull in admin journal
 - templating and css corrections, fixed repo switcher on chrome, updated titles
 - admin menu accessible from options menu on repository view
 - permissions cached queries
 
 1.0.0rc4  (**2010-10-12**)
 ==========================
 
 - fixed python2.5 missing simplejson imports (thanks to Jens Bäckman)
 - removed cache_manager settings from sqlalchemy meta
 - added sqlalchemy cache settings to ini files
 - validated password length and added second try of failure on paster setup-app
 - fixed setup database destroy prompt even when there was no db
 
 
 1.0.0rc3 (**2010-10-11**)
 =========================
 
 - fixed i18n during installation.
 
 1.0.0rc2 (**2010-10-11**)
 =========================
 
 - Disabled dirsize in file browser, it's causing nasty bug when dir renames 
   occure. After vcs is fixed it'll be put back again.
 - templating/css rewrites, optimized css.
  No newline at end of file

rhodecode/__init__.py

@@ -1,56 +1,56 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.__init__
     ~~~~~~~~~~~~~~~~~~
 
     RhodeCode, a web based repository management based on pylons
     versioning implementation: http://semver.org/
 
     :created_on: Apr 9, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import platform
 
-VERSION = (1, 2, 0, 'beta')
+VERSION = (1, 3, 0, 'beta')
 __version__ = '.'.join((str(each) for each in VERSION[:4]))
-__dbversion__ = 3 #defines current db version for migrations
+__dbversion__ = 4 #defines current db version for migrations
 __platform__ = platform.system()
 __license__ = 'GPLv3'
 
 PLATFORM_WIN = ('Windows')
 PLATFORM_OTHERS = ('Linux', 'Darwin', 'FreeBSD', 'OpenBSD', 'SunOS')
 
 try:
     from rhodecode.lib.utils import get_current_revision
     _rev = get_current_revision()
 except ImportError:
     #this is needed when doing some setup.py operations
     _rev = False
 
 if len(VERSION) > 3 and _rev:
     __version__ += ' [rev:%s]' % _rev[0]
 
 
 def get_version():
     """Returns shorter version (digit parts only) as string."""
 
     return '.'.join((str(each) for each in VERSION[:3]))
 
 BACKENDS = {
     'hg': 'Mercurial repository',
     #'git': 'Git repository',
 }

rhodecode/controllers/api/__init__.py

@@ -1,241 +1,242 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.api
     ~~~~~~~~~~~~~~~~~~~~~~~~~
 
     JSON RPC controller
     
     :created_on: Aug 20, 2011
     :author: marcink
     :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
 import inspect
-import json
 import logging
 import types
 import urllib
 import traceback
-from itertools import izip_longest
+
+from rhodecode.lib.compat import izip_longest, json
 
 from paste.response import replace_header
 
 from pylons.controllers import WSGIController
 from pylons.controllers.util import Response
 
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError, \
 HTTPBadRequest, HTTPError
 
 from rhodecode.model.db import User
 from rhodecode.lib.auth import AuthUser
 
 log = logging.getLogger('JSONRPC')
 
 class JSONRPCError(BaseException):
 
     def __init__(self, message):
         self.message = message
 
     def __str__(self):
         return str(self.message)
 
 
 def jsonrpc_error(message, code=None):
     """Generate a Response object with a JSON-RPC error body"""
     return Response(body=json.dumps(dict(result=None,
                                          error=message)))
 
 
 class JSONRPCController(WSGIController):
     """
      A WSGI-speaking JSON-RPC controller class
     
      See the specification:
      <http://json-rpc.org/wiki/specification>`.
    
      Valid controller return values should be json-serializable objects.
     
      Sub-classes should catch their exceptions and raise JSONRPCError
      if they want to pass meaningful errors to the client.
     
      """
 
     def _get_method_args(self):
         """
         Return `self._rpc_args` to dispatched controller method
         chosen by __call__
         """
         return self._rpc_args
 
     def __call__(self, environ, start_response):
         """
         Parse the request body as JSON, look up the method on the
         controller and if it exists, dispatch to it.
         """
         if 'CONTENT_LENGTH' not in environ:
             log.debug("No Content-Length")
             return jsonrpc_error(message="No Content-Length in request")
         else:
             length = environ['CONTENT_LENGTH'] or 0
             length = int(environ['CONTENT_LENGTH'])
             log.debug('Content-Length: %s', length)
 
         if length == 0:
             log.debug("Content-Length is 0")
             return jsonrpc_error(message="Content-Length is 0")
 
         raw_body = environ['wsgi.input'].read(length)
 
         try:
             json_body = json.loads(urllib.unquote_plus(raw_body))
-        except ValueError as e:
+        except ValueError, e:
             #catch JSON errors Here
             return jsonrpc_error(message="JSON parse error ERR:%s RAW:%r" \
                                  % (e, urllib.unquote_plus(raw_body)))
 
         #check AUTH based on API KEY
         try:
             self._req_api_key = json_body['api_key']
             self._req_method = json_body['method']
             self._req_params = json_body['args']
             log.debug('method: %s, params: %s',
                       self._req_method,
                       self._req_params)
-        except KeyError as e:
+        except KeyError, e:
             return jsonrpc_error(message='Incorrect JSON query missing %s' % e)
 
         #check if we can find this session using api_key
         try:
             u = User.get_by_api_key(self._req_api_key)
             auth_u = AuthUser(u.user_id, self._req_api_key)
-        except Exception as e:
+        except Exception, e:
             return jsonrpc_error(message='Invalid API KEY')
 
         self._error = None
         try:
             self._func = self._find_method()
         except AttributeError, e:
             return jsonrpc_error(message=str(e))
 
         # now that we have a method, add self._req_params to
         # self.kargs and dispatch control to WGIController
         argspec = inspect.getargspec(self._func)
         arglist = argspec[0][1:]
         defaults = argspec[3] or []
         default_empty = types.NotImplementedType
-        
-        kwarglist = list(izip_longest(reversed(arglist),reversed(defaults),
+
+        kwarglist = list(izip_longest(reversed(arglist), reversed(defaults),
                                 fillvalue=default_empty))
 
         # this is little trick to inject logged in user for 
         # perms decorators to work they expect the controller class to have
         # rhodecode_user attribute set
         self.rhodecode_user = auth_u
 
         # This attribute will need to be first param of a method that uses
         # api_key, which is translated to instance of user at that name
         USER_SESSION_ATTR = 'apiuser'
 
         if USER_SESSION_ATTR not in arglist:
             return jsonrpc_error(message='This method [%s] does not support '
                                  'authentication (missing %s param)' %
                                  (self._func.__name__, USER_SESSION_ATTR))
 
         # get our arglist and check if we provided them as args
-        for arg,default in kwarglist:
+        for arg, default in kwarglist:
             if arg == USER_SESSION_ATTR:
                 # USER_SESSION_ATTR is something translated from api key and 
                 # this is checked before so we don't need validate it
                 continue
-            
+
             # skip the required param check if it's default value is 
             # NotImplementedType (default_empty)
             if not self._req_params or (type(default) == default_empty
                                         and arg not in self._req_params):
                 return jsonrpc_error(message=('Missing non optional %s arg '
                                               'in JSON DATA') % arg)
 
         self._rpc_args = {USER_SESSION_ATTR:u}
         self._rpc_args.update(self._req_params)
 
         self._rpc_args['action'] = self._req_method
         self._rpc_args['environ'] = environ
         self._rpc_args['start_response'] = start_response
 
         status = []
         headers = []
         exc_info = []
         def change_content(new_status, new_headers, new_exc_info=None):
             status.append(new_status)
             headers.extend(new_headers)
             exc_info.append(new_exc_info)
 
         output = WSGIController.__call__(self, environ, change_content)
         output = list(output)
         headers.append(('Content-Length', str(len(output[0]))))
         replace_header(headers, 'Content-Type', 'application/json')
         start_response(status[0], headers, exc_info[0])
 
         return output
 
     def _dispatch_call(self):
         """
         Implement dispatch interface specified by WSGIController
         """
         try:
             raw_response = self._inspect_call(self._func)
             if isinstance(raw_response, HTTPError):
                 self._error = str(raw_response)
-        except JSONRPCError as e:
+        except JSONRPCError, e:
             self._error = str(e)
-        except Exception as e:
-            log.error('Encountered unhandled exception: %s' % traceback.format_exc())
+        except Exception, e:
+            log.error('Encountered unhandled exception: %s' \
+                      % traceback.format_exc())
             json_exc = JSONRPCError('Internal server error')
             self._error = str(json_exc)
 
         if self._error is not None:
             raw_response = None
 
         response = dict(result=raw_response, error=self._error)
 
         try:
             return json.dumps(response)
         except TypeError, e:
             log.debug('Error encoding response: %s', e)
             return json.dumps(dict(result=None,
                                    error="Error encoding response"))
 
     def _find_method(self):
         """
         Return method named by `self._req_method` in controller if able
         """
         log.debug('Trying to find JSON-RPC method: %s', self._req_method)
         if self._req_method.startswith('_'):
             raise AttributeError("Method not allowed")
 
         try:
             func = getattr(self, self._req_method, None)
         except UnicodeEncodeError:
             raise AttributeError("Problem decoding unicode in requested "
                                  "method name.")
 
         if isinstance(func, types.MethodType):
             return func
         else:
             raise AttributeError("No such method: %s" % self._req_method)
 

rhodecode/controllers/branches.py

@@ -1,78 +1,78 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.branches
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     branches controller for rhodecode
 
     :created_on: Apr 21, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
 
 from pylons import tmpl_context as c
 import binascii
 
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
-from rhodecode.lib.odict import OrderedDict
+from rhodecode.lib.compat import OrderedDict
 from rhodecode.lib import safe_unicode
 log = logging.getLogger(__name__)
 
 
 class BranchesController(BaseRepoController):
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def __before__(self):
         super(BranchesController, self).__before__()
 
     def index(self):
 
         def _branchtags(localrepo):
 
             bt = {}
             bt_closed = {}
 
             for bn, heads in localrepo.branchmap().iteritems():
                 tip = heads[-1]
                 if 'close' not in localrepo.changelog.read(tip)[5]:
                     bt[bn] = tip
                 else:
                     bt_closed[bn] = tip
             return bt, bt_closed
 
 
         bt, bt_closed = _branchtags(c.rhodecode_repo._repo)
         cs_g = c.rhodecode_repo.get_changeset
         _branches = [(safe_unicode(n), cs_g(binascii.hexlify(h)),) for n, h in
                      bt.items()]
 
         _closed_branches = [(safe_unicode(n), cs_g(binascii.hexlify(h)),) for n, h in
                      bt_closed.items()]
 
         c.repo_branches = OrderedDict(sorted(_branches,
                                              key=lambda ctx: ctx[0],
                                              reverse=False))
         c.repo_closed_branches = OrderedDict(sorted(_closed_branches,
                                                     key=lambda ctx: ctx[0],
                                                     reverse=False))
 
 
         return render('branches/branches.html')

rhodecode/controllers/changelog.py

@@ -1,121 +1,116 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.changelog
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     changelog controller for rhodecode
 
     :created_on: Apr 21, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
 
-try:
-    import json
-except ImportError:
-    #python 2.5 compatibility
-    import simplejson as json
-
 from mercurial import graphmod
 from pylons import request, session, tmpl_context as c
 
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.helpers import RepoPage
+from rhodecode.lib.compat import json
 
 log = logging.getLogger(__name__)
 
 
 class ChangelogController(BaseRepoController):
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def __before__(self):
         super(ChangelogController, self).__before__()
         c.affected_files_cut_off = 60
 
     def index(self):
         limit = 100
         default = 20
         if request.params.get('size'):
             try:
                 int_size = int(request.params.get('size'))
             except ValueError:
                 int_size = default
             int_size = int_size if int_size <= limit else limit
             c.size = int_size
             session['changelog_size'] = c.size
             session.save()
         else:
             c.size = int(session.get('changelog_size', default))
 
         p = int(request.params.get('page', 1))
         branch_name = request.params.get('branch', None)
         c.total_cs = len(c.rhodecode_repo)
         c.pagination = RepoPage(c.rhodecode_repo, page=p,
                                 item_count=c.total_cs, items_per_page=c.size,
                                 branch_name=branch_name)
 
         self._graph(c.rhodecode_repo, c.total_cs, c.size, p)
 
         return render('changelog/changelog.html')
 
     def changelog_details(self, cs):
         if request.environ.get('HTTP_X_PARTIAL_XHR'):
             c.cs = c.rhodecode_repo.get_changeset(cs)
             return render('changelog/changelog_details.html')
 
     def _graph(self, repo, repo_size, size, p):
         """
         Generates a DAG graph for mercurial
 
         :param repo: repo instance
         :param size: number of commits to show
         :param p: page number
         """
         if not repo.revisions:
             c.jsdata = json.dumps([])
             return
 
         revcount = min(repo_size, size)
         offset = 1 if p == 1 else  ((p - 1) * revcount + 1)
         try:
             rev_end = repo.revisions.index(repo.revisions[(-1 * offset)])
         except IndexError:
             rev_end = repo.revisions.index(repo.revisions[-1])
         rev_start = max(0, rev_end - revcount)
 
         data = []
         rev_end += 1
 
         if repo.alias == 'git':
             for _ in xrange(rev_start, rev_end):
                 vtx = [0, 1]
                 edges = [[0, 0, 1]]
                 data.append(['', vtx, edges])
 
         elif repo.alias == 'hg':
             revs = list(reversed(xrange(rev_start, rev_end)))
             c.dag = graphmod.colored(graphmod.dagwalker(repo._repo, revs))
             for (id, type, ctx, vtx, edges) in c.dag:
                 if type != graphmod.CHANGESET:
                     continue
                 data.append(['', vtx, edges])
 
         c.jsdata = json.dumps(data)

rhodecode/controllers/changeset.py

@@ -1,252 +1,252 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.changeset
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     changeset controller for pylons showoing changes beetween
     revisions
 
     :created_on: Apr 25, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import logging
 import traceback
 
 from pylons import tmpl_context as c, url, request, response
 from pylons.i18n.translation import _
 from pylons.controllers.util import redirect
 
 import rhodecode.lib.helpers as h
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.utils import EmptyChangeset
-from rhodecode.lib.odict import OrderedDict
+from rhodecode.lib.compat import OrderedDict
 
 from vcs.exceptions import RepositoryError, ChangesetError, \
 ChangesetDoesNotExistError
 from vcs.nodes import FileNode
 from vcs.utils import diffs as differ
 
 log = logging.getLogger(__name__)
 
 
 class ChangesetController(BaseRepoController):
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def __before__(self):
         super(ChangesetController, self).__before__()
         c.affected_files_cut_off = 60
 
     def index(self, revision):
 
         def wrap_to_table(str):
 
             return '''<table class="code-difftable">
                         <tr class="line">
                         <td class="lineno new"></td>
                         <td class="code"><pre>%s</pre></td>
                         </tr>
                       </table>''' % str
 
         #get ranges of revisions if preset
         rev_range = revision.split('...')[:2]
 
         try:
             if len(rev_range) == 2:
                 rev_start = rev_range[0]
                 rev_end = rev_range[1]
                 rev_ranges = c.rhodecode_repo.get_changesets(start=rev_start,
                                                             end=rev_end)
             else:
                 rev_ranges = [c.rhodecode_repo.get_changeset(revision)]
 
             c.cs_ranges = list(rev_ranges)
 
         except (RepositoryError, ChangesetDoesNotExistError, Exception), e:
             log.error(traceback.format_exc())
             h.flash(str(e), category='warning')
             return redirect(url('home'))
 
         c.changes = OrderedDict()
         c.sum_added = 0
         c.sum_removed = 0
         c.lines_added = 0
         c.lines_deleted = 0
         c.cut_off = False  # defines if cut off limit is reached
 
         # Iterate over ranges (default changeset view is always one changeset)
         for changeset in c.cs_ranges:
             c.changes[changeset.raw_id] = []
             try:
                 changeset_parent = changeset.parents[0]
             except IndexError:
                 changeset_parent = None
 
             #==================================================================
             # ADDED FILES
             #==================================================================
             for node in changeset.added:
 
                 filenode_old = FileNode(node.path, '', EmptyChangeset())
                 if filenode_old.is_binary or node.is_binary:
                     diff = wrap_to_table(_('binary file'))
                     st = (0, 0)
                 else:
                     # in this case node.size is good parameter since those are
                     # added nodes and their size defines how many changes were
                     # made
                     c.sum_added += node.size
                     if c.sum_added < self.cut_off_limit:
                         f_gitdiff = differ.get_gitdiff(filenode_old, node)
                         d = differ.DiffProcessor(f_gitdiff, format='gitdiff')
 
                         st = d.stat()
                         diff = d.as_html()
 
                     else:
                         diff = wrap_to_table(_('Changeset is to big and '
                                                'was cut off, see raw '
                                                'changeset instead'))
                         c.cut_off = True
                         break
 
                 cs1 = None
                 cs2 = node.last_changeset.raw_id
                 c.lines_added += st[0]
                 c.lines_deleted += st[1]
                 c.changes[changeset.raw_id].append(('added', node, diff,
                                                     cs1, cs2, st))
 
             #==================================================================
             # CHANGED FILES
             #==================================================================
             if not c.cut_off:
                 for node in changeset.changed:
                     try:
                         filenode_old = changeset_parent.get_node(node.path)
                     except ChangesetError:
                         log.warning('Unable to fetch parent node for diff')
                         filenode_old = FileNode(node.path, '',
                                                 EmptyChangeset())
 
                     if filenode_old.is_binary or node.is_binary:
                         diff = wrap_to_table(_('binary file'))
                         st = (0, 0)
                     else:
 
                         if c.sum_removed < self.cut_off_limit:
                             f_gitdiff = differ.get_gitdiff(filenode_old, node)
                             d = differ.DiffProcessor(f_gitdiff,
                                                      format='gitdiff')
                             st = d.stat()
                             if (st[0] + st[1]) * 256 > self.cut_off_limit:
                                 diff = wrap_to_table(_('Diff is to big '
                                                        'and was cut off, see '
                                                        'raw diff instead'))
                             else:
                                 diff = d.as_html()
 
                             if diff:
                                 c.sum_removed += len(diff)
                         else:
                             diff = wrap_to_table(_('Changeset is to big and '
                                                    'was cut off, see raw '
                                                    'changeset instead'))
                             c.cut_off = True
                             break
 
                     cs1 = filenode_old.last_changeset.raw_id
                     cs2 = node.last_changeset.raw_id
                     c.lines_added += st[0]
                     c.lines_deleted += st[1]
                     c.changes[changeset.raw_id].append(('changed', node, diff,
                                                         cs1, cs2, st))
 
             #==================================================================
             # REMOVED FILES
             #==================================================================
             if not c.cut_off:
                 for node in changeset.removed:
                     c.changes[changeset.raw_id].append(('removed', node, None,
                                                         None, None, (0, 0)))
 
         if len(c.cs_ranges) == 1:
             c.changeset = c.cs_ranges[0]
             c.changes = c.changes[c.changeset.raw_id]
 
             return render('changeset/changeset.html')
         else:
             return render('changeset/changeset_range.html')
 
     def raw_changeset(self, revision):
 
         method = request.GET.get('diff', 'show')
         try:
             c.scm_type = c.rhodecode_repo.alias
             c.changeset = c.rhodecode_repo.get_changeset(revision)
         except RepositoryError:
             log.error(traceback.format_exc())
             return redirect(url('home'))
         else:
             try:
                 c.changeset_parent = c.changeset.parents[0]
             except IndexError:
                 c.changeset_parent = None
             c.changes = []
 
             for node in c.changeset.added:
                 filenode_old = FileNode(node.path, '')
                 if filenode_old.is_binary or node.is_binary:
                     diff = _('binary file') + '\n'
                 else:
                     f_gitdiff = differ.get_gitdiff(filenode_old, node)
                     diff = differ.DiffProcessor(f_gitdiff,
                                                 format='gitdiff').raw_diff()
 
                 cs1 = None
                 cs2 = node.last_changeset.raw_id
                 c.changes.append(('added', node, diff, cs1, cs2))
 
             for node in c.changeset.changed:
                 filenode_old = c.changeset_parent.get_node(node.path)
                 if filenode_old.is_binary or node.is_binary:
                     diff = _('binary file')
                 else:
                     f_gitdiff = differ.get_gitdiff(filenode_old, node)
                     diff = differ.DiffProcessor(f_gitdiff,
                                                 format='gitdiff').raw_diff()
 
                 cs1 = filenode_old.last_changeset.raw_id
                 cs2 = node.last_changeset.raw_id
                 c.changes.append(('changed', node, diff, cs1, cs2))
 
         response.content_type = 'text/plain'
 
         if method == 'download':
             response.content_disposition = 'attachment; filename=%s.patch' \
                                             % revision
 
         c.parent_tmpl = ''.join(['# Parent  %s\n' % x.raw_id for x in
                                                  c.changeset.parents])
 
         c.diffs = ''
         for x in c.changes:
             c.diffs += x[2]
 
         return render('changeset/raw_changeset.html')

rhodecode/controllers/files.py

@@ -1,507 +1,514 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.files
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Files controller for RhodeCode
 
     :created_on: Apr 21, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import logging
 import traceback
 
 from os.path import join as jn
 
 from pylons import request, response, session, tmpl_context as c, url
 from pylons.i18n.translation import _
 from pylons.controllers.util import redirect
 from pylons.decorators import jsonify
 
 from vcs.conf import settings
 from vcs.exceptions import RepositoryError, ChangesetDoesNotExistError, \
     EmptyRepositoryError, ImproperArchiveTypeError, VCSError, NodeAlreadyExistsError
 from vcs.nodes import FileNode, NodeKind
 from vcs.utils import diffs as differ
 
 from rhodecode.lib import convert_line_endings, detect_mode, safe_str
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.utils import EmptyChangeset
 import rhodecode.lib.helpers as h
 from rhodecode.model.repo import RepoModel
 
 log = logging.getLogger(__name__)
 
 
 class FilesController(BaseRepoController):
 
     @LoginRequired()
     def __before__(self):
         super(FilesController, self).__before__()
         c.cut_off_limit = self.cut_off_limit
 
     def __get_cs_or_redirect(self, rev, repo_name, redirect_after=True):
         """
         Safe way to get changeset if error occur it redirects to tip with
         proper message
 
         :param rev: revision to fetch
         :param repo_name: repo name to redirect after
         """
 
         try:
             return c.rhodecode_repo.get_changeset(rev)
         except EmptyRepositoryError, e:
             if not redirect_after:
                 return None
             url_ = url('files_add_home',
                        repo_name=c.repo_name,
                        revision=0, f_path='')
             add_new = '<a href="%s">[%s]</a>' % (url_, _('add new'))
             h.flash(h.literal(_('There are no files yet %s' % add_new)),
                     category='warning')
             redirect(h.url('summary_home', repo_name=repo_name))
 
         except RepositoryError, e:
             h.flash(str(e), category='warning')
             redirect(h.url('files_home', repo_name=repo_name, revision='tip'))
 
     def __get_filenode_or_redirect(self, repo_name, cs, path):
         """
         Returns file_node, if error occurs or given path is directory,
         it'll redirect to top level path
 
         :param repo_name: repo_name
         :param cs: given changeset
         :param path: path to lookup
         """
 
         try:
             file_node = cs.get_node(path)
             if file_node.is_dir():
                 raise RepositoryError('given path is a directory')
         except RepositoryError, e:
             h.flash(str(e), category='warning')
             redirect(h.url('files_home', repo_name=repo_name,
                            revision=cs.raw_id))
 
         return file_node
 
 
     def __get_paths(self, changeset, starting_path):
         """recursive walk in root dir and return a set of all path in that dir
         based on repository walk function
         """
         _files = list()
         _dirs = list()
 
         try:
             tip = changeset
             for topnode, dirs, files in tip.walk(starting_path):
                 for f in files:
                     _files.append(f.path)
                 for d in dirs:
                     _dirs.append(d.path)
         except RepositoryError, e:
             log.debug(traceback.format_exc())
             pass
         return _dirs, _files
 
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def index(self, repo_name, revision, f_path):
         #reditect to given revision from form if given
         post_revision = request.POST.get('at_rev', None)
         if post_revision:
             cs = self.__get_cs_or_redirect(post_revision, repo_name)
             redirect(url('files_home', repo_name=c.repo_name,
                          revision=cs.raw_id, f_path=f_path))
 
         c.changeset = self.__get_cs_or_redirect(revision, repo_name)
         c.branch = request.GET.get('branch', None)
         c.f_path = f_path
 
         cur_rev = c.changeset.revision
 
         #prev link
         try:
             prev_rev = c.rhodecode_repo.get_changeset(cur_rev).prev(c.branch)
             c.url_prev = url('files_home', repo_name=c.repo_name,
                          revision=prev_rev.raw_id, f_path=f_path)
             if c.branch:
                 c.url_prev += '?branch=%s' % c.branch
         except (ChangesetDoesNotExistError, VCSError):
             c.url_prev = '#'
 
         #next link
         try:
             next_rev = c.rhodecode_repo.get_changeset(cur_rev).next(c.branch)
             c.url_next = url('files_home', repo_name=c.repo_name,
                      revision=next_rev.raw_id, f_path=f_path)
             if c.branch:
                 c.url_next += '?branch=%s' % c.branch
         except (ChangesetDoesNotExistError, VCSError):
             c.url_next = '#'
 
         #files or dirs
         try:
             c.files_list = c.changeset.get_node(f_path)
 
             if c.files_list.is_file():
                 c.file_history = self._get_node_history(c.changeset, f_path)
             else:
                 c.file_history = []
         except RepositoryError, e:
             h.flash(str(e), category='warning')
             redirect(h.url('files_home', repo_name=repo_name,
                            revision=revision))
 
         return render('files/files.html')
 
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def rawfile(self, repo_name, revision, f_path):
         cs = self.__get_cs_or_redirect(revision, repo_name)
         file_node = self.__get_filenode_or_redirect(repo_name, cs, f_path)
 
         response.content_disposition = 'attachment; filename=%s' % \
             safe_str(f_path.split(os.sep)[-1])
 
         response.content_type = file_node.mimetype
         return file_node.content
 
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def raw(self, repo_name, revision, f_path):
         cs = self.__get_cs_or_redirect(revision, repo_name)
         file_node = self.__get_filenode_or_redirect(repo_name, cs, f_path)
 
         raw_mimetype_mapping = {
             # map original mimetype to a mimetype used for "show as raw"
             # you can also provide a content-disposition to override the
             # default "attachment" disposition.
             # orig_type: (new_type, new_dispo)
 
             # show images inline:
             'image/x-icon': ('image/x-icon', 'inline'),
             'image/png': ('image/png', 'inline'),
             'image/gif': ('image/gif', 'inline'),
             'image/jpeg': ('image/jpeg', 'inline'),
             'image/svg+xml': ('image/svg+xml', 'inline'),
         }
 
         mimetype = file_node.mimetype
         try:
             mimetype, dispo = raw_mimetype_mapping[mimetype]
         except KeyError:
             # we don't know anything special about this, handle it safely
             if file_node.is_binary:
                 # do same as download raw for binary files
                 mimetype, dispo = 'application/octet-stream', 'attachment'
             else:
                 # do not just use the original mimetype, but force text/plain,
                 # otherwise it would serve text/html and that might be unsafe.
                 # Note: underlying vcs library fakes text/plain mimetype if the
                 # mimetype can not be determined and it thinks it is not
                 # binary.This might lead to erroneous text display in some
                 # cases, but helps in other cases, like with text files
                 # without extension.
                 mimetype, dispo = 'text/plain', 'inline'
 
         if dispo == 'attachment':
             dispo = 'attachment; filename=%s' % \
                         safe_str(f_path.split(os.sep)[-1])
 
         response.content_disposition = dispo
         response.content_type = mimetype
         return file_node.content
 
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def annotate(self, repo_name, revision, f_path):
         c.cs = self.__get_cs_or_redirect(revision, repo_name)
         c.file = self.__get_filenode_or_redirect(repo_name, c.cs, f_path)
 
         c.file_history = self._get_node_history(c.cs, f_path)
         c.f_path = f_path
         return render('files/files_annotate.html')
 
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     def edit(self, repo_name, revision, f_path):
         r_post = request.POST
 
         c.cs = self.__get_cs_or_redirect(revision, repo_name)
         c.file = self.__get_filenode_or_redirect(repo_name, c.cs, f_path)
 
         if c.file.is_binary:
             return redirect(url('files_home', repo_name=c.repo_name,
                          revision=c.cs.raw_id, f_path=f_path))
 
         c.f_path = f_path
 
         if r_post:
 
             old_content = c.file.content
             sl = old_content.splitlines(1)
             first_line = sl[0] if sl else ''
             # modes:  0 - Unix, 1 - Mac, 2 - DOS
             mode = detect_mode(first_line, 0)
             content = convert_line_endings(r_post.get('content'), mode)
 
             message = r_post.get('message') or (_('Edited %s via RhodeCode')
                                                 % (f_path))
             author = self.rhodecode_user.full_contact
 
             if content == old_content:
                 h.flash(_('No changes'),
                     category='warning')
                 return redirect(url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
 
             try:
                 self.scm_model.commit_change(repo=c.rhodecode_repo,
                                              repo_name=repo_name, cs=c.cs,
                                              user=self.rhodecode_user,
                                              author=author, message=message,
                                              content=content, f_path=f_path)
                 h.flash(_('Successfully committed to %s' % f_path),
                         category='success')
 
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred during commit'), category='error')
             return redirect(url('changeset_home',
                                 repo_name=c.repo_name, revision='tip'))
 
         return render('files/files_edit.html')
 
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     def add(self, repo_name, revision, f_path):
         r_post = request.POST
         c.cs = self.__get_cs_or_redirect(revision, repo_name,
                                          redirect_after=False)
         if c.cs is None:
             c.cs = EmptyChangeset(alias=c.rhodecode_repo.alias)
 
         c.f_path = f_path
 
         if r_post:
             unix_mode = 0
             content = convert_line_endings(r_post.get('content'), unix_mode)
 
             message = r_post.get('message') or (_('Added %s via RhodeCode')
                                                 % (f_path))
             location = r_post.get('location')
             filename = r_post.get('filename')
             file_obj = r_post.get('upload_file', None)
 
             if file_obj is not None and hasattr(file_obj, 'filename'):
                 filename = file_obj.filename
                 content = file_obj.file
 
+            #TODO: REMOVE THIS !!
+            ################################
+            import ipdb;ipdb.set_trace()
+            print 'setting ipdb debuggin for rhodecode.controllers.files.FilesController.add'
+            ################################
+
+
             node_path = os.path.join(location, filename)
             author = self.rhodecode_user.full_contact
 
             if not content:
                 h.flash(_('No content'), category='warning')
                 return redirect(url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
             if not filename:
                 h.flash(_('No filename'), category='warning')
                 return redirect(url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
 
             try:
                 self.scm_model.create_node(repo=c.rhodecode_repo,
                                              repo_name=repo_name, cs=c.cs,
                                              user=self.rhodecode_user,
                                              author=author, message=message,
                                              content=content, f_path=node_path)
                 h.flash(_('Successfully committed to %s' % node_path),
                         category='success')
             except NodeAlreadyExistsError, e:
                 h.flash(_(e), category='error')
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred during commit'), category='error')
             return redirect(url('changeset_home',
                                 repo_name=c.repo_name, revision='tip'))
 
         return render('files/files_add.html')
 
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def archivefile(self, repo_name, fname):
 
         fileformat = None
         revision = None
         ext = None
         subrepos = request.GET.get('subrepos') == 'true'
 
         for a_type, ext_data in settings.ARCHIVE_SPECS.items():
             archive_spec = fname.split(ext_data[1])
             if len(archive_spec) == 2 and archive_spec[1] == '':
                 fileformat = a_type or ext_data[1]
                 revision = archive_spec[0]
                 ext = ext_data[1]
 
         try:
             dbrepo = RepoModel().get_by_repo_name(repo_name)
             if dbrepo.enable_downloads is False:
                 return _('downloads disabled')
 
             cs = c.rhodecode_repo.get_changeset(revision)
             content_type = settings.ARCHIVE_SPECS[fileformat][0]
         except ChangesetDoesNotExistError:
             return _('Unknown revision %s') % revision
         except EmptyRepositoryError:
             return _('Empty repository')
         except (ImproperArchiveTypeError, KeyError):
             return _('Unknown archive type')
 
         response.content_type = content_type
         response.content_disposition = 'attachment; filename=%s-%s%s' \
             % (repo_name, revision, ext)
 
         import tempfile
         archive = tempfile.mkstemp()[1]
         t = open(archive, 'wb')
         cs.fill_archive(stream=t, kind=fileformat, subrepos=subrepos)
 
         def get_chunked_archive(archive):
             stream = open(archive, 'rb')
             while True:
                 data = stream.read(4096)
                 if not data:
                     os.remove(archive)
                     break
                 yield data
 
         return get_chunked_archive(archive)
 
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def diff(self, repo_name, f_path):
         diff1 = request.GET.get('diff1')
         diff2 = request.GET.get('diff2')
         c.action = request.GET.get('diff')
         c.no_changes = diff1 == diff2
         c.f_path = f_path
         c.big_diff = False
 
         try:
             if diff1 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_1 = c.rhodecode_repo.get_changeset(diff1)
                 node1 = c.changeset_1.get_node(f_path)
             else:
                 c.changeset_1 = EmptyChangeset(repo=c.rhodecode_repo)
                 node1 = FileNode('.', '', changeset=c.changeset_1)
 
             if diff2 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_2 = c.rhodecode_repo.get_changeset(diff2)
                 node2 = c.changeset_2.get_node(f_path)
             else:
                 c.changeset_2 = EmptyChangeset(repo=c.rhodecode_repo)
                 node2 = FileNode('.', '', changeset=c.changeset_2)
         except RepositoryError:
             return redirect(url('files_home',
                                 repo_name=c.repo_name, f_path=f_path))
 
         if c.action == 'download':
             diff = differ.DiffProcessor(differ.get_gitdiff(node1, node2),
                                         format='gitdiff')
 
             diff_name = '%s_vs_%s.diff' % (diff1, diff2)
             response.content_type = 'text/plain'
             response.content_disposition = 'attachment; filename=%s' \
                                                     % diff_name
             return diff.raw_diff()
 
         elif c.action == 'raw':
             diff = differ.DiffProcessor(differ.get_gitdiff(node1, node2),
                                         format='gitdiff')
             response.content_type = 'text/plain'
             return diff.raw_diff()
 
         elif c.action == 'diff':
             if node1.is_binary or node2.is_binary:
                 c.cur_diff = _('Binary file')
             elif node1.size > self.cut_off_limit or \
                     node2.size > self.cut_off_limit:
                 c.cur_diff = ''
                 c.big_diff = True
             else:
                 diff = differ.DiffProcessor(differ.get_gitdiff(node1, node2),
                                         format='gitdiff')
                 c.cur_diff = diff.as_html()
         else:
 
             #default option
             if node1.is_binary or node2.is_binary:
                 c.cur_diff = _('Binary file')
             elif node1.size > self.cut_off_limit or \
                     node2.size > self.cut_off_limit:
                 c.cur_diff = ''
                 c.big_diff = True
 
             else:
                 diff = differ.DiffProcessor(differ.get_gitdiff(node1, node2),
                                         format='gitdiff')
                 c.cur_diff = diff.as_html()
 
         if not c.cur_diff and not c.big_diff:
             c.no_changes = True
         return render('files/file_diff.html')
 
     def _get_node_history(self, cs, f_path):
         changesets = cs.get_file_history(f_path)
         hist_l = []
 
         changesets_group = ([], _("Changesets"))
         branches_group = ([], _("Branches"))
         tags_group = ([], _("Tags"))
 
         for chs in changesets:
             n_desc = 'r%s:%s' % (chs.revision, chs.short_id)
             changesets_group[0].append((chs.raw_id, n_desc,))
 
         hist_l.append(changesets_group)
 
         for name, chs in c.rhodecode_repo.branches.items():
             #chs = chs.split(':')[-1]
             branches_group[0].append((chs, name),)
         hist_l.append(branches_group)
 
         for name, chs in c.rhodecode_repo.tags.items():
             #chs = chs.split(':')[-1]
             tags_group[0].append((chs, name),)
         hist_l.append(tags_group)
 
         return hist_l
 
     @jsonify
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def nodelist(self, repo_name, revision, f_path):
         if request.environ.get('HTTP_X_PARTIAL_XHR'):
             cs = self.__get_cs_or_redirect(revision, repo_name)
             _d, _f = self.__get_paths(cs, f_path)
             return _d + _f
 

rhodecode/controllers/summary.py

@@ -1,189 +1,183 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.summary
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Summary controller for Rhodecode
 
     :created_on: Apr 18, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import calendar
 import logging
 from time import mktime
 from datetime import datetime, timedelta, date
 
 from vcs.exceptions import ChangesetError
 
 from pylons import tmpl_context as c, request, url
 from pylons.i18n.translation import _
 
 from rhodecode.model.db import Statistics, Repository
 from rhodecode.model.repo import RepoModel
 
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.utils import EmptyChangeset
-from rhodecode.lib.odict import OrderedDict
 
 from rhodecode.lib.celerylib import run_task
 from rhodecode.lib.celerylib.tasks import get_commits_stats, \
     LANGUAGES_EXTENSIONS_MAP
 from rhodecode.lib.helpers import RepoPage
+from rhodecode.lib.compat import json, OrderedDict
 
-try:
-    import json
-except ImportError:
-    #python 2.5 compatibility
-    import simplejson as json
-    
 log = logging.getLogger(__name__)
 
 
 class SummaryController(BaseRepoController):
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def __before__(self):
         super(SummaryController, self).__before__()
 
     def index(self, repo_name):
 
         e = request.environ
         c.dbrepo = dbrepo = c.rhodecode_db_repo
 
         c.following = self.scm_model.is_following_repo(repo_name,
                                                 self.rhodecode_user.user_id)
 
         def url_generator(**kw):
             return url('shortlog_home', repo_name=repo_name, size=10, **kw)
 
         c.repo_changesets = RepoPage(c.rhodecode_repo, page=1,
                                      items_per_page=10, url=url_generator)
 
         if self.rhodecode_user.username == 'default':
             #for default(anonymous) user we don't need to pass credentials
             username = ''
             password = ''
         else:
             username = str(self.rhodecode_user.username)
             password = '@'
 
         if e.get('wsgi.url_scheme') == 'https':
             split_s = 'https://'
         else:
             split_s = 'http://'
 
         qualified_uri = [split_s] + [url.current(qualified=True)\
                                      .split(split_s)[-1]]
         uri = u'%(proto)s%(user)s%(pass)s%(rest)s' \
                 % {'user': username,
                      'pass': password,
                      'proto': qualified_uri[0],
                      'rest': qualified_uri[1]}
         c.clone_repo_url = uri
         c.repo_tags = OrderedDict()
         for name, hash in c.rhodecode_repo.tags.items()[:10]:
             try:
                 c.repo_tags[name] = c.rhodecode_repo.get_changeset(hash)
             except ChangesetError:
                 c.repo_tags[name] = EmptyChangeset(hash)
 
         c.repo_branches = OrderedDict()
         for name, hash in c.rhodecode_repo.branches.items()[:10]:
             try:
                 c.repo_branches[name] = c.rhodecode_repo.get_changeset(hash)
             except ChangesetError:
                 c.repo_branches[name] = EmptyChangeset(hash)
 
         td = date.today() + timedelta(days=1)
         td_1m = td - timedelta(days=calendar.mdays[td.month])
         td_1y = td - timedelta(days=365)
 
         ts_min_m = mktime(td_1m.timetuple())
         ts_min_y = mktime(td_1y.timetuple())
         ts_max_y = mktime(td.timetuple())
 
         if dbrepo.enable_statistics:
             c.no_data_msg = _('No data loaded yet')
             run_task(get_commits_stats, c.dbrepo.repo_name, ts_min_y, ts_max_y)
         else:
             c.no_data_msg = _('Statistics are disabled for this repository')
         c.ts_min = ts_min_m
         c.ts_max = ts_max_y
 
         stats = self.sa.query(Statistics)\
             .filter(Statistics.repository == dbrepo)\
             .scalar()
 
         c.stats_percentage = 0
 
         if stats and stats.languages:
             c.no_data = False is dbrepo.enable_statistics
             lang_stats_d = json.loads(stats.languages)
             c.commit_data = stats.commit_activity
             c.overview_data = stats.commit_activity_combined
-            
+
             lang_stats = ((x, {"count": y,
                                "desc": LANGUAGES_EXTENSIONS_MAP.get(x)})
                           for x, y in lang_stats_d.items())
 
             c.trending_languages = json.dumps(OrderedDict(
                                        sorted(lang_stats, reverse=True,
                                             key=lambda k: k[1])[:10]
                                         )
                                     )
             last_rev = stats.stat_on_revision
             c.repo_last_rev = c.rhodecode_repo.count() - 1 \
                 if c.rhodecode_repo.revisions else 0
             if last_rev == 0 or c.repo_last_rev == 0:
                 pass
             else:
                 c.stats_percentage = '%.2f' % ((float((last_rev)) /
                                                 c.repo_last_rev) * 100)
         else:
             c.commit_data = json.dumps({})
             c.overview_data = json.dumps([[ts_min_y, 0], [ts_max_y, 10]])
             c.trending_languages = json.dumps({})
             c.no_data = True
-            
+
         c.enable_downloads = dbrepo.enable_downloads
         if c.enable_downloads:
             c.download_options = self._get_download_links(c.rhodecode_repo)
 
         return render('summary/summary.html')
 
     def _get_download_links(self, repo):
 
         download_l = []
 
         branches_group = ([], _("Branches"))
         tags_group = ([], _("Tags"))
 
         for name, chs in c.rhodecode_repo.branches.items():
             #chs = chs.split(':')[-1]
             branches_group[0].append((chs, name),)
         download_l.append(branches_group)
 
         for name, chs in c.rhodecode_repo.tags.items():
             #chs = chs.split(':')[-1]
             tags_group[0].append((chs, name),)
         download_l.append(tags_group)
 
         return download_l

rhodecode/controllers/tags.py

@@ -1,53 +1,53 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.tags
     ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Tags controller for rhodecode
 
     :created_on: Apr 21, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import logging
 
 from pylons import tmpl_context as c
 
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
-from rhodecode.lib.odict import OrderedDict
+from rhodecode.lib.compat import OrderedDict
 
 log = logging.getLogger(__name__)
 
 
 class TagsController(BaseRepoController):
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def __before__(self):
         super(TagsController, self).__before__()
 
     def index(self):
         c.repo_tags = OrderedDict()
 
         tags = [(name, c.rhodecode_repo.get_changeset(hash_)) for \
                  name, hash_ in c.rhodecode_repo.tags.items()]
         ordered_tags = sorted(tags, key=lambda x: x[1].date, reverse=True)
         for name, cs_tag in ordered_tags:
             c.repo_tags[name] = cs_tag
 
         return render('tags/tags.html')

rhodecode/lib/__init__.py

@@ -1,389 +1,381 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.__init__
     ~~~~~~~~~~~~~~~~~~~~~~~
 
     Some simple helper functions
 
     :created_on: Jan 5, 2011
     :author: marcink
     :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
-try:
-    import json
-except ImportError:
-    #python 2.5 compatibility
-    import simplejson as json
-
-
 def __get_lem():
     from pygments import lexers
     from string import lower
     from collections import defaultdict
 
     d = defaultdict(lambda: [])
 
     def __clean(s):
         s = s.lstrip('*')
         s = s.lstrip('.')
 
         if s.find('[') != -1:
             exts = []
             start, stop = s.find('['), s.find(']')
 
             for suffix in s[start + 1:stop]:
                 exts.append(s[:s.find('[')] + suffix)
             return map(lower, exts)
         else:
             return map(lower, [s])
 
     for lx, t in sorted(lexers.LEXERS.items()):
         m = map(__clean, t[-2])
         if m:
             m = reduce(lambda x, y: x + y, m)
             for ext in m:
                 desc = lx.replace('Lexer', '')
                 d[ext].append(desc)
 
     return dict(d)
 
 # language map is also used by whoosh indexer, which for those specified
 # extensions will index it's content
 LANGUAGES_EXTENSIONS_MAP = __get_lem()
 
 # Additional mappings that are not present in the pygments lexers
 # NOTE: that this will overide any mappings in LANGUAGES_EXTENSIONS_MAP
 ADDITIONAL_MAPPINGS = {'xaml': 'XAML'}
 
 LANGUAGES_EXTENSIONS_MAP.update(ADDITIONAL_MAPPINGS)
 
 
 def str2bool(_str):
     """
     returs True/False value from given string, it tries to translate the
     string into boolean
 
     :param _str: string value to translate into boolean
     :rtype: boolean
     :returns: boolean from given string
     """
     if _str is None:
         return False
     if _str in (True, False):
         return _str
     _str = str(_str).strip().lower()
     return _str in ('t', 'true', 'y', 'yes', 'on', '1')
 
 
 def convert_line_endings(line, mode):
     """
     Converts a given line  "line end" accordingly to given mode
     
     Available modes are::
         0 - Unix
         1 - Mac
         2 - DOS
     
     :param line: given line to convert
     :param mode: mode to convert to
     :rtype: str
     :return: converted line according to mode
     """
     from string import replace
 
     if mode == 0:
             line = replace(line, '\r\n', '\n')
             line = replace(line, '\r', '\n')
     elif mode == 1:
             line = replace(line, '\r\n', '\r')
             line = replace(line, '\n', '\r')
     elif mode == 2:
             import re
             line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line)
     return line
 
 
 def detect_mode(line, default):
     """
     Detects line break for given line, if line break couldn't be found
     given default value is returned
 
     :param line: str line
     :param default: default
     :rtype: int
     :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
     """
     if line.endswith('\r\n'):
         return 2
     elif line.endswith('\n'):
         return 0
     elif line.endswith('\r'):
         return 1
     else:
         return default
 
 
 def generate_api_key(username, salt=None):
     """
     Generates unique API key for given username, if salt is not given
     it'll be generated from some random string
 
     :param username: username as string
     :param salt: salt to hash generate KEY
     :rtype: str
     :returns: sha1 hash from username+salt
     """
     from tempfile import _RandomNameSequence
     import hashlib
 
     if salt is None:
         salt = _RandomNameSequence().next()
 
     return hashlib.sha1(username + salt).hexdigest()
 
 
 def safe_unicode(str_, from_encoding='utf8'):
     """
     safe unicode function. Does few trick to turn str_ into unicode
      
     In case of UnicodeDecode error we try to return it with encoding detected
     by chardet library if it fails fallback to unicode with errors replaced
 
     :param str_: string to decode
     :rtype: unicode
     :returns: unicode object
     """
     if isinstance(str_, unicode):
         return str_
 
     try:
         return unicode(str_)
     except UnicodeDecodeError:
         pass
 
     try:
         return unicode(str_, from_encoding)
     except UnicodeDecodeError:
         pass
 
     try:
         import chardet
         encoding = chardet.detect(str_)['encoding']
         if encoding is None:
             raise Exception()
         return str_.decode(encoding)
     except (ImportError, UnicodeDecodeError, Exception):
         return unicode(str_, from_encoding, 'replace')
 
 def safe_str(unicode_, to_encoding='utf8'):
     """
     safe str function. Does few trick to turn unicode_ into string
      
     In case of UnicodeEncodeError we try to return it with encoding detected
     by chardet library if it fails fallback to string with errors replaced
 
     :param unicode_: unicode to encode
     :rtype: str
     :returns: str object
     """
 
     if isinstance(unicode_, str):
         return unicode_
 
     try:
         return unicode_.encode(to_encoding)
     except UnicodeEncodeError:
         pass
-    
+
     try:
         import chardet
         encoding = chardet.detect(unicode_)['encoding']
         print encoding
         if encoding is None:
             raise UnicodeEncodeError()
-        
+
         return unicode_.encode(encoding)
     except (ImportError, UnicodeEncodeError):
         return unicode_.encode(to_encoding, 'replace')
 
     return safe_str
 
 
 
 def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
     """
     Custom engine_from_config functions that makes sure we use NullPool for
     file based sqlite databases. This prevents errors on sqlite. This only 
     applies to sqlalchemy versions < 0.7.0
 
     """
     import sqlalchemy
     from sqlalchemy import engine_from_config as efc
     import logging
 
     if int(sqlalchemy.__version__.split('.')[1]) < 7:
 
         # This solution should work for sqlalchemy < 0.7.0, and should use
         # proxy=TimerProxy() for execution time profiling
 
         from sqlalchemy.pool import NullPool
         url = configuration[prefix + 'url']
 
         if url.startswith('sqlite'):
             kwargs.update({'poolclass': NullPool})
         return efc(configuration, prefix, **kwargs)
     else:
         import time
         from sqlalchemy import event
         from sqlalchemy.engine import Engine
 
         log = logging.getLogger('sqlalchemy.engine')
         BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38)
         engine = efc(configuration, prefix, **kwargs)
 
         def color_sql(sql):
             COLOR_SEQ = "\033[1;%dm"
             COLOR_SQL = YELLOW
             normal = '\x1b[0m'
             return ''.join([COLOR_SEQ % COLOR_SQL, sql, normal])
 
         if configuration['debug']:
             #attach events only for debug configuration
 
             def before_cursor_execute(conn, cursor, statement,
                                     parameters, context, executemany):
                 context._query_start_time = time.time()
                 log.info(color_sql(">>>>> STARTING QUERY >>>>>"))
 
 
             def after_cursor_execute(conn, cursor, statement,
                                     parameters, context, executemany):
                 total = time.time() - context._query_start_time
                 log.info(color_sql("<<<<< TOTAL TIME: %f <<<<<" % total))
 
             event.listen(engine, "before_cursor_execute",
                          before_cursor_execute)
             event.listen(engine, "after_cursor_execute",
                          after_cursor_execute)
 
     return engine
 
 
 def age(curdate):
     """
     turns a datetime into an age string.
     
     :param curdate: datetime object
     :rtype: unicode
     :returns: unicode words describing age
     """
 
     from datetime import datetime
     from webhelpers.date import time_ago_in_words
 
     _ = lambda s:s
 
     if not curdate:
         return ''
 
     agescales = [(_(u"year"), 3600 * 24 * 365),
                  (_(u"month"), 3600 * 24 * 30),
                  (_(u"day"), 3600 * 24),
                  (_(u"hour"), 3600),
                  (_(u"minute"), 60),
                  (_(u"second"), 1), ]
 
     age = datetime.now() - curdate
     age_seconds = (age.days * agescales[2][1]) + age.seconds
     pos = 1
     for scale in agescales:
         if scale[1] <= age_seconds:
             if pos == 6:pos = 5
             return '%s %s' % (time_ago_in_words(curdate,
                                                 agescales[pos][0]), _('ago'))
         pos += 1
 
     return _(u'just now')
 
 
 def uri_filter(uri):
     """
     Removes user:password from given url string
     
     :param uri:
     :rtype: unicode
     :returns: filtered list of strings  
     """
     if not uri:
         return ''
 
     proto = ''
 
     for pat in ('https://', 'http://'):
         if uri.startswith(pat):
             uri = uri[len(pat):]
             proto = pat
             break
 
     # remove passwords and username
     uri = uri[uri.find('@') + 1:]
 
     # get the port
     cred_pos = uri.find(':')
     if cred_pos == -1:
         host, port = uri, None
     else:
         host, port = uri[:cred_pos], uri[cred_pos + 1:]
 
     return filter(None, [proto, host, port])
 
 
 def credentials_filter(uri):
     """
     Returns a url with removed credentials
     
     :param uri:
     """
 
     uri = uri_filter(uri)
     #check if we have port
     if len(uri) > 2 and uri[2]:
         uri[2] = ':' + uri[2]
 
     return ''.join(uri)
 
 def get_changeset_safe(repo, rev):
     """
     Safe version of get_changeset if this changeset doesn't exists for a 
     repo it returns a Dummy one instead
     
     :param repo:
     :param rev:
     """
     from vcs.backends.base import BaseRepository
     from vcs.exceptions import RepositoryError
     if not isinstance(repo, BaseRepository):
         raise Exception('You must pass an Repository '
                         'object as first argument got %s', type(repo))
 
     try:
         cs = repo.get_changeset(rev)
     except RepositoryError:
         from rhodecode.lib.utils import EmptyChangeset
         cs = EmptyChangeset(requested_revision=rev)
-    return cs
  No newline at end of file
+    return cs

rhodecode/lib/celerylib/tasks.py

@@ -1,413 +1,410 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.celerylib.tasks
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     RhodeCode task modules, containing all task that suppose to be run
     by celery daemon
 
     :created_on: Oct 6, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 from celery.decorators import task
 
 import os
 import traceback
 import logging
 from os.path import dirname as dn, join as jn
 
 from time import mktime
 from operator import itemgetter
 from string import lower
 
 from pylons import config, url
 from pylons.i18n.translation import _
 
 from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP, safe_str
 from rhodecode.lib.celerylib import run_task, locked_task, str2bool, \
     __get_lockkey, LockHeld, DaemonLock
 from rhodecode.lib.helpers import person
 from rhodecode.lib.smtp_mailer import SmtpMailer
 from rhodecode.lib.utils import add_cache
-from rhodecode.lib.odict import OrderedDict
+from rhodecode.lib.compat import json, OrderedDict
+
 from rhodecode.model import init_model
 from rhodecode.model import meta
 from rhodecode.model.db import RhodeCodeUi, Statistics, Repository
 
 from vcs.backends import get_repo
 
 from sqlalchemy import engine_from_config
 
 add_cache(config)
 
-try:
-    import json
-except ImportError:
-    #python 2.5 compatibility
-    import simplejson as json
+
 
 __all__ = ['whoosh_index', 'get_commits_stats',
            'reset_user_password', 'send_email']
 
 CELERY_ON = str2bool(config['app_conf'].get('use_celery'))
 
 
 def get_session():
     if CELERY_ON:
         engine = engine_from_config(config, 'sqlalchemy.db1.')
         init_model(engine)
     sa = meta.Session()
     return sa
 
 
 def get_repos_path():
     sa = get_session()
     q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
     return q.ui_value
 
 
 @task(ignore_result=True)
 @locked_task
 def whoosh_index(repo_location, full_index):
     #log = whoosh_index.get_logger()
     from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
     index_location = config['index_dir']
     WhooshIndexingDaemon(index_location=index_location,
                          repo_location=repo_location, sa=get_session())\
                          .run(full_index=full_index)
 
 
 @task(ignore_result=True)
 def get_commits_stats(repo_name, ts_min_y, ts_max_y):
     try:
         log = get_commits_stats.get_logger()
     except:
         log = logging.getLogger(__name__)
 
     lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y,
                             ts_max_y)
     lockkey_path = dn(dn(dn(dn(os.path.abspath(__file__)))))
     log.info('running task with lockkey %s', lockkey)
     try:
         lock = l = DaemonLock(jn(lockkey_path, lockkey))
 
         #for js data compatibilty cleans the key for person from '
         akc = lambda k: person(k).replace('"', "")
 
         co_day_auth_aggr = {}
         commits_by_day_aggregate = {}
         repos_path = get_repos_path()
         repo = get_repo(safe_str(os.path.join(repos_path, repo_name)))
         repo_size = len(repo.revisions)
         #return if repo have no revisions
         if repo_size < 1:
             lock.release()
             return True
 
         skip_date_limit = True
         parse_limit = int(config['app_conf'].get('commit_parse_limit'))
         last_rev = 0
         last_cs = None
         timegetter = itemgetter('time')
 
         sa = get_session()
 
         dbrepo = sa.query(Repository)\
             .filter(Repository.repo_name == repo_name).scalar()
         cur_stats = sa.query(Statistics)\
             .filter(Statistics.repository == dbrepo).scalar()
 
         if cur_stats is not None:
             last_rev = cur_stats.stat_on_revision
 
         if last_rev == repo.get_changeset().revision and repo_size > 1:
             #pass silently without any work if we're not on first revision or
             #current state of parsing revision(from db marker) is the
             #last revision
             lock.release()
             return True
 
         if cur_stats:
             commits_by_day_aggregate = OrderedDict(json.loads(
                                         cur_stats.commit_activity_combined))
             co_day_auth_aggr = json.loads(cur_stats.commit_activity)
 
         log.debug('starting parsing %s', parse_limit)
         lmktime = mktime
 
         last_rev = last_rev + 1 if last_rev > 0 else last_rev
 
         for cs in repo[last_rev:last_rev + parse_limit]:
             last_cs = cs  # remember last parsed changeset
             k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1],
                           cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0])
 
             if akc(cs.author) in co_day_auth_aggr:
                 try:
                     l = [timegetter(x) for x in
                          co_day_auth_aggr[akc(cs.author)]['data']]
                     time_pos = l.index(k)
                 except ValueError:
                     time_pos = False
 
                 if time_pos >= 0 and time_pos is not False:
 
                     datadict = \
                         co_day_auth_aggr[akc(cs.author)]['data'][time_pos]
 
                     datadict["commits"] += 1
                     datadict["added"] += len(cs.added)
                     datadict["changed"] += len(cs.changed)
                     datadict["removed"] += len(cs.removed)
 
                 else:
                     if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
 
                         datadict = {"time": k,
                                     "commits": 1,
                                     "added": len(cs.added),
                                     "changed": len(cs.changed),
                                     "removed": len(cs.removed),
                                    }
                         co_day_auth_aggr[akc(cs.author)]['data']\
                             .append(datadict)
 
             else:
                 if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
                     co_day_auth_aggr[akc(cs.author)] = {
                                         "label": akc(cs.author),
                                         "data": [{"time":k,
                                                  "commits":1,
                                                  "added":len(cs.added),
                                                  "changed":len(cs.changed),
                                                  "removed":len(cs.removed),
                                                  }],
                                         "schema": ["commits"],
                                         }
 
             #gather all data by day
             if k in commits_by_day_aggregate:
                 commits_by_day_aggregate[k] += 1
             else:
                 commits_by_day_aggregate[k] = 1
 
         overview_data = sorted(commits_by_day_aggregate.items(),
                                key=itemgetter(0))
 
         if not co_day_auth_aggr:
             co_day_auth_aggr[akc(repo.contact)] = {
                 "label": akc(repo.contact),
                 "data": [0, 1],
                 "schema": ["commits"],
             }
 
         stats = cur_stats if cur_stats else Statistics()
         stats.commit_activity = json.dumps(co_day_auth_aggr)
         stats.commit_activity_combined = json.dumps(overview_data)
 
         log.debug('last revison %s', last_rev)
         leftovers = len(repo.revisions[last_rev:])
         log.debug('revisions to parse %s', leftovers)
 
         if last_rev == 0 or leftovers < parse_limit:
             log.debug('getting code trending stats')
             stats.languages = json.dumps(__get_codes_stats(repo_name))
 
         try:
             stats.repository = dbrepo
             stats.stat_on_revision = last_cs.revision if last_cs else 0
             sa.add(stats)
             sa.commit()
         except:
             log.error(traceback.format_exc())
             sa.rollback()
             lock.release()
             return False
 
         #final release
         lock.release()
 
         #execute another task if celery is enabled
         if len(repo.revisions) > 1 and CELERY_ON:
             run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
         return True
     except LockHeld:
         log.info('LockHeld')
         return 'Task with key %s already running' % lockkey
 
 @task(ignore_result=True)
 def send_password_link(user_email):
     try:
         log = reset_user_password.get_logger()
     except:
         log = logging.getLogger(__name__)
 
     from rhodecode.lib import auth
     from rhodecode.model.db import User
 
     try:
         sa = get_session()
         user = sa.query(User).filter(User.email == user_email).scalar()
 
         if user:
             link = url('reset_password_confirmation', key=user.api_key,
                        qualified=True)
             tmpl = """
 Hello %s
 
 We received a request to create a new password for your account.
 
 You can generate it by clicking following URL:
 
 %s
 
 If you didn't request new password please ignore this email.
             """
             run_task(send_email, user_email,
                      "RhodeCode password reset link",
                      tmpl % (user.short_contact, link))
             log.info('send new password mail to %s', user_email)
 
     except:
         log.error('Failed to update user password')
         log.error(traceback.format_exc())
         return False
 
     return True
 
 @task(ignore_result=True)
 def reset_user_password(user_email):
     try:
         log = reset_user_password.get_logger()
     except:
         log = logging.getLogger(__name__)
 
     from rhodecode.lib import auth
     from rhodecode.model.db import User
 
     try:
         try:
             sa = get_session()
             user = sa.query(User).filter(User.email == user_email).scalar()
             new_passwd = auth.PasswordGenerator().gen_password(8,
                              auth.PasswordGenerator.ALPHABETS_BIG_SMALL)
             if user:
                 user.password = auth.get_crypt_password(new_passwd)
                 user.api_key = auth.generate_api_key(user.username)
                 sa.add(user)
                 sa.commit()
                 log.info('change password for %s', user_email)
             if new_passwd is None:
                 raise Exception('unable to generate new password')
 
         except:
             log.error(traceback.format_exc())
             sa.rollback()
 
         run_task(send_email, user_email,
                  "Your new RhodeCode password",
                  'Your new RhodeCode password:%s' % (new_passwd))
         log.info('send new password mail to %s', user_email)
 
     except:
         log.error('Failed to update user password')
         log.error(traceback.format_exc())
 
     return True
 
 
 @task(ignore_result=True)
 def send_email(recipients, subject, body):
     """
     Sends an email with defined parameters from the .ini files.
 
     :param recipients: list of recipients, it this is empty the defined email
         address from field 'email_to' is used instead
     :param subject: subject of the mail
     :param body: body of the mail
     """
     try:
         log = send_email.get_logger()
     except:
         log = logging.getLogger(__name__)
 
     email_config = config
 
     if not recipients:
         recipients = [email_config.get('email_to')]
 
     mail_from = email_config.get('app_email_from')
     user = email_config.get('smtp_username')
     passwd = email_config.get('smtp_password')
     mail_server = email_config.get('smtp_server')
     mail_port = email_config.get('smtp_port')
     tls = str2bool(email_config.get('smtp_use_tls'))
     ssl = str2bool(email_config.get('smtp_use_ssl'))
     debug = str2bool(config.get('debug'))
 
     try:
         m = SmtpMailer(mail_from, user, passwd, mail_server,
                        mail_port, ssl, tls, debug=debug)
         m.send(recipients, subject, body)
     except:
         log.error('Mail sending failed')
         log.error(traceback.format_exc())
         return False
     return True
 
 
 @task(ignore_result=True)
 def create_repo_fork(form_data, cur_user):
     from rhodecode.model.repo import RepoModel
     from vcs import get_backend
 
     try:
         log = create_repo_fork.get_logger()
     except:
         log = logging.getLogger(__name__)
 
     repo_model = RepoModel(get_session())
     repo_model.create(form_data, cur_user, just_db=True, fork=True)
     repo_name = form_data['repo_name']
     repos_path = get_repos_path()
     repo_path = os.path.join(repos_path, repo_name)
     repo_fork_path = os.path.join(repos_path, form_data['fork_name'])
     alias = form_data['repo_type']
 
     log.info('creating repo fork %s as %s', repo_name, repo_path)
     backend = get_backend(alias)
     backend(str(repo_fork_path), create=True, src_url=str(repo_path))
 
 
 def __get_codes_stats(repo_name):
     repos_path = get_repos_path()
     repo = get_repo(safe_str(os.path.join(repos_path, repo_name)))
     tip = repo.get_changeset()
     code_stats = {}
 
     def aggregate(cs):
         for f in cs[2]:
             ext = lower(f.extension)
             if ext in LANGUAGES_EXTENSIONS_MAP.keys() and not f.is_binary:
                 if ext in code_stats:
                     code_stats[ext] += 1
                 else:
                     code_stats[ext] = 1
 
     map(aggregate, tip.walk('/'))
 
     return code_stats or {}

rhodecode/model/db.py

@@ -1,1003 +1,1004 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.model.db
     ~~~~~~~~~~~~~~~~~~
 
     Database Models for RhodeCode
 
     :created_on: Apr 08, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import logging
 import datetime
 import traceback
 from datetime import date
 
 from sqlalchemy import *
 from sqlalchemy.exc import DatabaseError
 from sqlalchemy.orm import relationship, backref, joinedload, class_mapper
 from sqlalchemy.orm.interfaces import MapperExtension
 
 from beaker.cache import cache_region, region_invalidate
 
 from vcs import get_backend
 from vcs.utils.helpers import get_scm
 from vcs.exceptions import VCSError
 from vcs.utils.lazy import LazyProperty
 
+from rhodecode.lib import str2bool, safe_str, get_changeset_safe,\
+    generate_api_key
 from rhodecode.lib.exceptions import UsersGroupsAssignedException
-from rhodecode.lib import str2bool, json, safe_str, get_changeset_safe, \
-    generate_api_key
+from rhodecode.lib.compat import json
 
 from rhodecode.model.meta import Base, Session
 from rhodecode.model.caching_query import FromCache
 
 log = logging.getLogger( __name__ )
 
 #==============================================================================
 # BASE CLASSES
 #==============================================================================
 
 class ModelSerializer( json.JSONEncoder ):
     """
     Simple Serializer for JSON,
 
     usage::
 
         to make object customized for serialization implement a __json__
         method that will return a dict for serialization into json
 
     example::
 
         class Task(object):
 
             def __init__(self, name, value):
                 self.name = name
                 self.value = value
 
             def __json__(self):
                 return dict(name=self.name,
                             value=self.value)
 
     """
 
     def default( self, obj ):
 
         if hasattr( obj, '__json__' ):
             return obj.__json__()
         else:
             return json.JSONEncoder.default( self, obj )
 
 class BaseModel( object ):
     """Base Model for all classess
 
     """
 
     @classmethod
     def _get_keys( cls ):
         """return column names for this model """
         return class_mapper( cls ).c.keys()
 
     def get_dict( self ):
         """return dict with keys and values corresponding
         to this model data """
 
         d = {}
         for k in self._get_keys():
             d[k] = getattr( self, k )
         return d
 
     def get_appstruct( self ):
         """return list with keys and values tupples corresponding
         to this model data """
 
         l = []
         for k in self._get_keys():
             l.append( ( k, getattr( self, k ), ) )
         return l
 
     def populate_obj( self, populate_dict ):
         """populate model with data from given populate_dict"""
 
         for k in self._get_keys():
             if k in populate_dict:
                 setattr( self, k, populate_dict[k] )
 
     @classmethod
     def query( cls ):
         return Session.query( cls )
 
     @classmethod
     def get( cls, id_ ):
         return cls.query().get( id_ )
 
     @classmethod
     def getAll( cls ):
         return cls.query().all()
 
     @classmethod
     def delete( cls, id_ ):
         obj = cls.query().get( id_ )
         Session.delete( obj )
         Session.commit()
 
 
 class RhodeCodeSettings( Base, BaseModel ):
     __tablename__ = 'rhodecode_settings'
     __table_args__ = ( UniqueConstraint( 'app_settings_name' ), {'extend_existing':True} )
     app_settings_id = Column( "app_settings_id", Integer(), nullable = False, unique = True, default = None, primary_key = True )
     app_settings_name = Column( "app_settings_name", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
     app_settings_value = Column( "app_settings_value", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
 
     def __init__( self, k = '', v = '' ):
         self.app_settings_name = k
         self.app_settings_value = v
 
     def __repr__( self ):
         return "<%s('%s:%s')>" % ( self.__class__.__name__,
                                   self.app_settings_name, self.app_settings_value )
 
 
     @classmethod
     def get_by_name( cls, ldap_key ):
         return cls.query()\
             .filter( cls.app_settings_name == ldap_key ).scalar()
 
     @classmethod
     def get_app_settings( cls, cache = False ):
 
         ret = cls.query()
 
         if cache:
             ret = ret.options( FromCache( "sql_cache_short", "get_hg_settings" ) )
 
         if not ret:
             raise Exception( 'Could not get application settings !' )
         settings = {}
         for each in ret:
             settings['rhodecode_' + each.app_settings_name] = \
                 each.app_settings_value
 
         return settings
 
     @classmethod
     def get_ldap_settings( cls, cache = False ):
         ret = cls.query()\
                 .filter( cls.app_settings_name.startswith( 'ldap_' ) )\
                 .all()
         fd = {}
         for row in ret:
             fd.update( {row.app_settings_name:row.app_settings_value} )
 
         fd.update( {'ldap_active':str2bool( fd.get( 'ldap_active' ) )} )
 
         return fd
 
 
 class RhodeCodeUi( Base, BaseModel ):
     __tablename__ = 'rhodecode_ui'
     __table_args__ = ( UniqueConstraint( 'ui_key' ), {'extend_existing':True} )
 
     HOOK_UPDATE = 'changegroup.update'
     HOOK_REPO_SIZE = 'changegroup.repo_size'
     HOOK_PUSH = 'pretxnchangegroup.push_logger'
     HOOK_PULL = 'preoutgoing.pull_logger'
 
     ui_id = Column( "ui_id", Integer(), nullable = False, unique = True, default = None, primary_key = True )
     ui_section = Column( "ui_section", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
     ui_key = Column( "ui_key", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
     ui_value = Column( "ui_value", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
     ui_active = Column( "ui_active", Boolean(), nullable = True, unique = None, default = True )
 
 
     @classmethod
     def get_by_key( cls, key ):
         return cls.query().filter( cls.ui_key == key )
 
 
     @classmethod
     def get_builtin_hooks( cls ):
         q = cls.query()
         q = q.filter( cls.ui_key.in_( [cls.HOOK_UPDATE,
                                     cls.HOOK_REPO_SIZE,
                                     cls.HOOK_PUSH, cls.HOOK_PULL] ) )
         return q.all()
 
     @classmethod
     def get_custom_hooks( cls ):
         q = cls.query()
         q = q.filter( ~cls.ui_key.in_( [cls.HOOK_UPDATE,
                                     cls.HOOK_REPO_SIZE,
                                     cls.HOOK_PUSH, cls.HOOK_PULL] ) )
         q = q.filter( cls.ui_section == 'hooks' )
         return q.all()
 
     @classmethod
     def create_or_update_hook( cls, key, val ):
         new_ui = cls.get_by_key( key ).scalar() or cls()
         new_ui.ui_section = 'hooks'
         new_ui.ui_active = True
         new_ui.ui_key = key
         new_ui.ui_value = val
 
         Session.add( new_ui )
         Session.commit()
 
 
 class User( Base, BaseModel ):
     __tablename__ = 'users'
     __table_args__ = ( UniqueConstraint( 'username' ), UniqueConstraint( 'email' ), {'extend_existing':True} )
     user_id = Column( "user_id", Integer(), nullable = False, unique = True, default = None, primary_key = True )
     username = Column( "username", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
     password = Column( "password", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
     active = Column( "active", Boolean(), nullable = True, unique = None, default = None )
     admin = Column( "admin", Boolean(), nullable = True, unique = None, default = False )
     name = Column( "name", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
     lastname = Column( "lastname", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
     email = Column( "email", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
     last_login = Column( "last_login", DateTime( timezone = False ), nullable = True, unique = None, default = None )
     ldap_dn = Column( "ldap_dn", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
     api_key = Column( "api_key", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
 
     user_log = relationship( 'UserLog', cascade = 'all' )
     user_perms = relationship( 'UserToPerm', primaryjoin = "User.user_id==UserToPerm.user_id", cascade = 'all' )
 
     repositories = relationship( 'Repository' )
     user_followers = relationship( 'UserFollowing', primaryjoin = 'UserFollowing.follows_user_id==User.user_id', cascade = 'all' )
     repo_to_perm = relationship( 'RepoToPerm', primaryjoin = 'RepoToPerm.user_id==User.user_id', cascade = 'all' )
 
     group_member = relationship( 'UsersGroupMember', cascade = 'all' )
 
     @property
     def full_contact( self ):
         return '%s %s <%s>' % ( self.name, self.lastname, self.email )
 
     @property
     def short_contact( self ):
         return '%s %s' % ( self.name, self.lastname )
 
     @property
     def is_admin( self ):
         return self.admin
 
     def __repr__( self ):
         try:
             return "<%s('id:%s:%s')>" % ( self.__class__.__name__,
                                              self.user_id, self.username )
         except:
             return self.__class__.__name__
 
     @classmethod
     def by_username( cls, username, case_insensitive = False ):
         if case_insensitive:
             return cls.query().filter( cls.username.like( username ) ).one()
         else:
             return cls.query().filter( cls.username == username ).one()
 
     @classmethod
     def get_by_api_key( cls, api_key ):
         return cls.query().filter( cls.api_key == api_key ).one()
 
     def update_lastlogin( self ):
         """Update user lastlogin"""
 
         self.last_login = datetime.datetime.now()
         Session.add( self )
         Session.commit()
         log.debug( 'updated user %s lastlogin', self.username )
 
     @classmethod
     def create( cls, form_data ):
         from rhodecode.lib.auth import get_crypt_password
 
         try:
             new_user = cls()
             for k, v in form_data.items():
                 if k == 'password':
                     v = get_crypt_password( v )
                 setattr( new_user, k, v )
 
             new_user.api_key = generate_api_key( form_data['username'] )
             Session.add( new_user )
             Session.commit()
             return new_user
         except:
             log.error( traceback.format_exc() )
             Session.rollback()
             raise
 
 class UserLog( Base, BaseModel ):
     __tablename__ = 'user_logs'
     __table_args__ = {'extend_existing':True}
     user_log_id = Column( "user_log_id", Integer(), nullable = False, unique = True, default = None, primary_key = True )
     user_id = Column( "user_id", Integer(), ForeignKey( 'users.user_id' ), nullable = False, unique = None, default = None )
     repository_id = Column( "repository_id", Integer(), ForeignKey( 'repositories.repo_id' ), nullable = False, unique = None, default = None )
     repository_name = Column( "repository_name", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
     user_ip = Column( "user_ip", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
     action = Column( "action", UnicodeText( length = 1200000, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
     action_date = Column( "action_date", DateTime( timezone = False ), nullable = True, unique = None, default = None )
 
     @property
     def action_as_day( self ):
         return date( *self.action_date.timetuple()[:3] )
 
     user = relationship( 'User' )
     repository = relationship( 'Repository' )
 
 
 class UsersGroup( Base, BaseModel ):
     __tablename__ = 'users_groups'
     __table_args__ = {'extend_existing':True}
 
     users_group_id = Column( "users_group_id", Integer(), nullable = False, unique = True, default = None, primary_key = True )
     users_group_name = Column( "users_group_name", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = False, unique = True, default = None )
     users_group_active = Column( "users_group_active", Boolean(), nullable = True, unique = None, default = None )
 
     members = relationship( 'UsersGroupMember', cascade = "all, delete, delete-orphan", lazy = "joined" )
 
     def __repr__( self ):
         return '<userGroup(%s)>' % ( self.users_group_name )
 
     @classmethod
     def get_by_group_name( cls, group_name, cache = False, case_insensitive = False ):
         if case_insensitive:
             gr = cls.query()\
                 .filter( cls.users_group_name.ilike( group_name ) )
         else:
             gr = cls.query()\
                 .filter( cls.users_group_name == group_name )
         if cache:
             gr = gr.options( FromCache( "sql_cache_short",
                                           "get_user_%s" % group_name ) )
         return gr.scalar()
 
 
     @classmethod
     def get( cls, users_group_id, cache = False ):
         users_group = cls.query()
         if cache:
             users_group = users_group.options( FromCache( "sql_cache_short",
                                     "get_users_group_%s" % users_group_id ) )
         return users_group.get( users_group_id )
 
     @classmethod
     def create( cls, form_data ):
         try:
             new_users_group = cls()
             for k, v in form_data.items():
                 setattr( new_users_group, k, v )
 
             Session.add( new_users_group )
             Session.commit()
             return new_users_group
         except:
             log.error( traceback.format_exc() )
             Session.rollback()
             raise
 
     @classmethod
     def update( cls, users_group_id, form_data ):
 
         try:
             users_group = cls.get( users_group_id, cache = False )
 
             for k, v in form_data.items():
                 if k == 'users_group_members':
                     users_group.members = []
                     Session.flush()
                     members_list = []
                     if v:
                         for u_id in set( v ):
                             members_list.append( UsersGroupMember( 
                                                             users_group_id,
                                                             u_id ) )
                     setattr( users_group, 'members', members_list )
                 setattr( users_group, k, v )
 
             Session.add( users_group )
             Session.commit()
         except:
             log.error( traceback.format_exc() )
             Session.rollback()
             raise
 
     @classmethod
     def delete( cls, users_group_id ):
         try:
 
             # check if this group is not assigned to repo
             assigned_groups = UsersGroupRepoToPerm.query()\
                 .filter( UsersGroupRepoToPerm.users_group_id ==
                         users_group_id ).all()
 
             if assigned_groups:
                 raise UsersGroupsAssignedException( 'Group assigned to %s' %
                                                    assigned_groups )
 
             users_group = cls.get( users_group_id, cache = False )
             Session.delete( users_group )
             Session.commit()
         except:
             log.error( traceback.format_exc() )
             Session.rollback()
             raise
 
 class UsersGroupMember( Base, BaseModel ):
     __tablename__ = 'users_groups_members'
     __table_args__ = {'extend_existing':True}
 
     users_group_member_id = Column( "users_group_member_id", Integer(), nullable = False, unique = True, default = None, primary_key = True )
     users_group_id = Column( "users_group_id", Integer(), ForeignKey( 'users_groups.users_group_id' ), nullable = False, unique = None, default = None )
     user_id = Column( "user_id", Integer(), ForeignKey( 'users.user_id' ), nullable = False, unique = None, default = None )
 
     user = relationship( 'User', lazy = 'joined' )
     users_group = relationship( 'UsersGroup' )
 
     def __init__( self, gr_id = '', u_id = '' ):
         self.users_group_id = gr_id
         self.user_id = u_id
 
     @staticmethod
     def add_user_to_group( group, user ):
         ugm = UsersGroupMember()
         ugm.users_group = group
         ugm.user = user
         Session.add( ugm )
         Session.commit()
         return ugm
 
 class Repository( Base, BaseModel ):
     __tablename__ = 'repositories'
     __table_args__ = ( UniqueConstraint( 'repo_name' ), {'extend_existing':True}, )
 
     repo_id = Column( "repo_id", Integer(), nullable = False, unique = True, default = None, primary_key = True )
     repo_name = Column( "repo_name", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = False, unique = True, default = None )
     clone_uri = Column( "clone_uri", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = False, default = None )
     repo_type = Column( "repo_type", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = False, unique = False, default = 'hg' )
     user_id = Column( "user_id", Integer(), ForeignKey( 'users.user_id' ), nullable = False, unique = False, default = None )
     private = Column( "private", Boolean(), nullable = True, unique = None, default = None )
     enable_statistics = Column( "statistics", Boolean(), nullable = True, unique = None, default = True )
     enable_downloads = Column( "downloads", Boolean(), nullable = True, unique = None, default = True )
     description = Column( "description", String( length = 10000, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
     created_on = Column( 'created_on', DateTime( timezone = False ), nullable = True, unique = None, default = datetime.datetime.now )
 
     fork_id = Column( "fork_id", Integer(), ForeignKey( 'repositories.repo_id' ), nullable = True, unique = False, default = None )
     group_id = Column( "group_id", Integer(), ForeignKey( 'groups.group_id' ), nullable = True, unique = False, default = None )
 
 
     user = relationship( 'User' )
     fork = relationship( 'Repository', remote_side = repo_id )
     group = relationship( 'Group' )
     repo_to_perm = relationship( 'RepoToPerm', cascade = 'all', order_by = 'RepoToPerm.repo_to_perm_id' )
     users_group_to_perm = relationship( 'UsersGroupRepoToPerm', cascade = 'all' )
     stats = relationship( 'Statistics', cascade = 'all', uselist = False )
 
     followers = relationship( 'UserFollowing', primaryjoin = 'UserFollowing.follows_repo_id==Repository.repo_id', cascade = 'all' )
 
     logs = relationship( 'UserLog', cascade = 'all' )
 
     def __repr__( self ):
         return "<%s('%s:%s')>" % ( self.__class__.__name__,
                                   self.repo_id, self.repo_name )
 
     @classmethod
     def by_repo_name( cls, repo_name ):
         q = cls.query().filter( cls.repo_name == repo_name )
 
         q = q.options( joinedload( Repository.fork ) )\
             .options( joinedload( Repository.user ) )\
             .options( joinedload( Repository.group ) )\
 
         return q.one()
 
     @classmethod
     def get_repo_forks( cls, repo_id ):
         return cls.query().filter( Repository.fork_id == repo_id )
 
     @classmethod
     def base_path( cls ):
         """
         Returns base path when all repos are stored
 
         :param cls:
         """
         q = Session.query( RhodeCodeUi ).filter( RhodeCodeUi.ui_key == '/' )
         q.options( FromCache( "sql_cache_short", "repository_repo_path" ) )
         return q.one().ui_value
 
     @property
     def just_name( self ):
         return self.repo_name.split( os.sep )[-1]
 
     @property
     def groups_with_parents( self ):
         groups = []
         if self.group is None:
             return groups
 
         cur_gr = self.group
         groups.insert( 0, cur_gr )
         while 1:
             gr = getattr( cur_gr, 'parent_group', None )
             cur_gr = cur_gr.parent_group
             if gr is None:
                 break
             groups.insert( 0, gr )
 
         return groups
 
     @property
     def groups_and_repo( self ):
         return self.groups_with_parents, self.just_name
 
     @LazyProperty
     def repo_path( self ):
         """
         Returns base full path for that repository means where it actually
         exists on a filesystem
         """
         q = RhodeCodeUi.query().filter( RhodeCodeUi.ui_key == '/' )
         q.options( FromCache( "sql_cache_short", "repository_repo_path" ) )
         return q.one().ui_value
 
     @property
     def repo_full_path( self ):
         p = [self.repo_path]
         # we need to split the name by / since this is how we store the
         # names in the database, but that eventually needs to be converted
         # into a valid system path
         p += self.repo_name.split( '/' )
         return os.path.join( *p )
 
     @property
     def _ui( self ):
         """
         Creates an db based ui object for this repository
         """
         from mercurial import ui
         from mercurial import config
         baseui = ui.ui()
 
         #clean the baseui object
         baseui._ocfg = config.config()
         baseui._ucfg = config.config()
         baseui._tcfg = config.config()
 
 
         ret = RhodeCodeUi.query()\
             .options( FromCache( "sql_cache_short", "repository_repo_ui" ) ).all()
 
         hg_ui = ret
         for ui_ in hg_ui:
             if ui_.ui_active:
                 log.debug( 'settings ui from db[%s]%s:%s', ui_.ui_section,
                           ui_.ui_key, ui_.ui_value )
                 baseui.setconfig( ui_.ui_section, ui_.ui_key, ui_.ui_value )
 
         return baseui
 
     @classmethod
     def is_valid( cls, repo_name ):
         """
         returns True if given repo name is a valid filesystem repository
 
         @param cls:
         @param repo_name:
         """
         from rhodecode.lib.utils import is_valid_repo
 
         return is_valid_repo( repo_name, cls.base_path() )
 
 
     #==========================================================================
     # SCM PROPERTIES
     #==========================================================================
 
     def get_changeset( self, rev ):
         return get_changeset_safe( self.scm_instance, rev )
 
     @property
     def tip( self ):
         return self.get_changeset( 'tip' )
 
     @property
     def author( self ):
         return self.tip.author
 
     @property
     def last_change( self ):
         return self.scm_instance.last_change
 
     #==========================================================================
     # SCM CACHE INSTANCE
     #==========================================================================
 
     @property
     def invalidate( self ):
         """
         Returns Invalidation object if this repo should be invalidated
         None otherwise. `cache_active = False` means that this cache
         state is not valid and needs to be invalidated
         """
         return CacheInvalidation.query()\
             .filter( CacheInvalidation.cache_key == self.repo_name )\
             .filter( CacheInvalidation.cache_active == False )\
             .scalar()
 
     def set_invalidate( self ):
         """
         set a cache for invalidation for this instance
         """
         inv = CacheInvalidation.query()\
             .filter( CacheInvalidation.cache_key == self.repo_name )\
             .scalar()
 
         if inv is None:
             inv = CacheInvalidation( self.repo_name )
         inv.cache_active = True
         Session.add( inv )
         Session.commit()
 
     @LazyProperty
     def scm_instance( self ):
         return self.__get_instance()
 
     @property
     def scm_instance_cached( self ):
         @cache_region( 'long_term' )
         def _c( repo_name ):
             return self.__get_instance()
 
         # TODO: remove this trick when beaker 1.6 is released
         # and have fixed this issue with not supporting unicode keys
         rn = safe_str( self.repo_name )
 
         inv = self.invalidate
         if inv is not None:
             region_invalidate( _c, None, rn )
             # update our cache
             inv.cache_active = True
             Session.add( inv )
             Session.commit()
 
         return _c( rn )
 
     def __get_instance( self ):
 
         repo_full_path = self.repo_full_path
 
         try:
             alias = get_scm( repo_full_path )[0]
             log.debug( 'Creating instance of %s repository', alias )
             backend = get_backend( alias )
         except VCSError:
             log.error( traceback.format_exc() )
             log.error( 'Perhaps this repository is in db and not in '
                       'filesystem run rescan repositories with '
                       '"destroy old data " option from admin panel' )
             return
 
         if alias == 'hg':
 
             repo = backend( safe_str( repo_full_path ), create = False,
                            baseui = self._ui )
             #skip hidden web repository
             if repo._get_hidden():
                 return
         else:
             repo = backend( repo_full_path, create = False )
 
         return repo
 
 
 class Group( Base, BaseModel ):
     __tablename__ = 'groups'
     __table_args__ = ( UniqueConstraint( 'group_name', 'group_parent_id' ),
                       CheckConstraint( 'group_id != group_parent_id' ), {'extend_existing':True}, )
     __mapper_args__ = {'order_by':'group_name'}
 
     group_id = Column( "group_id", Integer(), nullable = False, unique = True, default = None, primary_key = True )
     group_name = Column( "group_name", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = False, unique = True, default = None )
     group_parent_id = Column( "group_parent_id", Integer(), ForeignKey( 'groups.group_id' ), nullable = True, unique = None, default = None )
     group_description = Column( "group_description", String( length = 10000, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
 
     parent_group = relationship( 'Group', remote_side = group_id )
 
 
     def __init__( self, group_name = '', parent_group = None ):
         self.group_name = group_name
         self.parent_group = parent_group
 
     def __repr__( self ):
         return "<%s('%s:%s')>" % ( self.__class__.__name__, self.group_id,
                                   self.group_name )
 
     @classmethod
     def url_sep( cls ):
         return '/'
 
     @classmethod
     def get_by_group_name( cls, group_name, cache = False, case_insensitive = False ):
         if case_insensitive:
             gr = cls.query()\
                 .filter( cls.group_name.ilike( group_name ) )
         else:
             gr = cls.query()\
                 .filter( cls.group_name == group_name )
         if cache:
             gr = gr.options( FromCache( "sql_cache_short",
                                           "get_group_%s" % group_name ) )
         return gr.scalar()
 
     @property
     def parents( self ):
         parents_recursion_limit = 5
         groups = []
         if self.parent_group is None:
             return groups
         cur_gr = self.parent_group
         groups.insert( 0, cur_gr )
         cnt = 0
         while 1:
             cnt += 1
             gr = getattr( cur_gr, 'parent_group', None )
             cur_gr = cur_gr.parent_group
             if gr is None:
                 break
             if cnt == parents_recursion_limit:
                 # this will prevent accidental infinit loops
                 log.error( 'group nested more than %s' %
                           parents_recursion_limit )
                 break
 
             groups.insert( 0, gr )
         return groups
 
     @property
     def children( self ):
         return Group.query().filter( Group.parent_group == self )
 
     @property
     def full_path( self ):
         return Group.url_sep().join( [g.group_name for g in self.parents] +
                         [self.group_name] )
 
     @property
     def repositories( self ):
         return Repository.query().filter( Repository.group == self )
 
     @property
     def repositories_recursive_count( self ):
         cnt = self.repositories.count()
 
         def children_count( group ):
             cnt = 0
             for child in group.children:
                 cnt += child.repositories.count()
                 cnt += children_count( child )
             return cnt
 
         return cnt + children_count( self )
 
 class Permission( Base, BaseModel ):
     __tablename__ = 'permissions'
     __table_args__ = {'extend_existing':True}
     permission_id = Column( "permission_id", Integer(), nullable = False, unique = True, default = None, primary_key = True )
     permission_name = Column( "permission_name", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
     permission_longname = Column( "permission_longname", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
 
     def __repr__( self ):
         return "<%s('%s:%s')>" % ( self.__class__.__name__,
                                   self.permission_id, self.permission_name )
 
     @classmethod
     def get_by_key( cls, key ):
         return cls.query().filter( cls.permission_name == key ).scalar()
 
 class RepoToPerm( Base, BaseModel ):
     __tablename__ = 'repo_to_perm'
     __table_args__ = ( UniqueConstraint( 'user_id', 'repository_id' ), {'extend_existing':True} )
     repo_to_perm_id = Column( "repo_to_perm_id", Integer(), nullable = False, unique = True, default = None, primary_key = True )
     user_id = Column( "user_id", Integer(), ForeignKey( 'users.user_id' ), nullable = False, unique = None, default = None )
     permission_id = Column( "permission_id", Integer(), ForeignKey( 'permissions.permission_id' ), nullable = False, unique = None, default = None )
     repository_id = Column( "repository_id", Integer(), ForeignKey( 'repositories.repo_id' ), nullable = False, unique = None, default = None )
 
     user = relationship( 'User' )
     permission = relationship( 'Permission' )
     repository = relationship( 'Repository' )
 
 class UserToPerm( Base, BaseModel ):
     __tablename__ = 'user_to_perm'
     __table_args__ = ( UniqueConstraint( 'user_id', 'permission_id' ), {'extend_existing':True} )
     user_to_perm_id = Column( "user_to_perm_id", Integer(), nullable = False, unique = True, default = None, primary_key = True )
     user_id = Column( "user_id", Integer(), ForeignKey( 'users.user_id' ), nullable = False, unique = None, default = None )
     permission_id = Column( "permission_id", Integer(), ForeignKey( 'permissions.permission_id' ), nullable = False, unique = None, default = None )
 
     user = relationship( 'User' )
     permission = relationship( 'Permission' )
 
     @classmethod
     def has_perm( cls, user_id, perm ):
         if not isinstance( perm, Permission ):
             raise Exception( 'perm needs to be an instance of Permission class' )
 
         return cls.query().filter( cls.user_id == user_id )\
             .filter( cls.permission == perm ).scalar() is not None
 
     @classmethod
     def grant_perm( cls, user_id, perm ):
         if not isinstance( perm, Permission ):
             raise Exception( 'perm needs to be an instance of Permission class' )
 
         new = cls()
         new.user_id = user_id
         new.permission = perm
         try:
             Session.add( new )
             Session.commit()
         except:
             Session.rollback()
 
 
     @classmethod
     def revoke_perm( cls, user_id, perm ):
         if not isinstance( perm, Permission ):
             raise Exception( 'perm needs to be an instance of Permission class' )
 
         try:
             cls.query().filter( cls.user_id == user_id )\
                 .filter( cls.permission == perm ).delete()
             Session.commit()
         except:
             Session.rollback()
 
 class UsersGroupRepoToPerm( Base, BaseModel ):
     __tablename__ = 'users_group_repo_to_perm'
     __table_args__ = ( UniqueConstraint( 'repository_id', 'users_group_id', 'permission_id' ), {'extend_existing':True} )
     users_group_to_perm_id = Column( "users_group_to_perm_id", Integer(), nullable = False, unique = True, default = None, primary_key = True )
     users_group_id = Column( "users_group_id", Integer(), ForeignKey( 'users_groups.users_group_id' ), nullable = False, unique = None, default = None )
     permission_id = Column( "permission_id", Integer(), ForeignKey( 'permissions.permission_id' ), nullable = False, unique = None, default = None )
     repository_id = Column( "repository_id", Integer(), ForeignKey( 'repositories.repo_id' ), nullable = False, unique = None, default = None )
 
     users_group = relationship( 'UsersGroup' )
     permission = relationship( 'Permission' )
     repository = relationship( 'Repository' )
 
     def __repr__( self ):
         return '<userGroup:%s => %s >' % ( self.users_group, self.repository )
 
 class UsersGroupToPerm( Base, BaseModel ):
     __tablename__ = 'users_group_to_perm'
     users_group_to_perm_id = Column( "users_group_to_perm_id", Integer(), nullable = False, unique = True, default = None, primary_key = True )
     users_group_id = Column( "users_group_id", Integer(), ForeignKey( 'users_groups.users_group_id' ), nullable = False, unique = None, default = None )
     permission_id = Column( "permission_id", Integer(), ForeignKey( 'permissions.permission_id' ), nullable = False, unique = None, default = None )
 
     users_group = relationship( 'UsersGroup' )
     permission = relationship( 'Permission' )
 
 
     @classmethod
     def has_perm( cls, users_group_id, perm ):
         if not isinstance( perm, Permission ):
             raise Exception( 'perm needs to be an instance of Permission class' )
 
         return cls.query().filter( cls.users_group_id ==
                                          users_group_id )\
                                          .filter( cls.permission == perm )\
                                          .scalar() is not None
 
     @classmethod
     def grant_perm( cls, users_group_id, perm ):
         if not isinstance( perm, Permission ):
             raise Exception( 'perm needs to be an instance of Permission class' )
 
         new = cls()
         new.users_group_id = users_group_id
         new.permission = perm
         try:
             Session.add( new )
             Session.commit()
         except:
             Session.rollback()
 
 
     @classmethod
     def revoke_perm( cls, users_group_id, perm ):
         if not isinstance( perm, Permission ):
             raise Exception( 'perm needs to be an instance of Permission class' )
 
         try:
             cls.query().filter( cls.users_group_id == users_group_id )\
                 .filter( cls.permission == perm ).delete()
             Session.commit()
         except:
             Session.rollback()
 
 
 class GroupToPerm( Base, BaseModel ):
     __tablename__ = 'group_to_perm'
     __table_args__ = ( UniqueConstraint( 'group_id', 'permission_id' ), {'extend_existing':True} )
 
     group_to_perm_id = Column( "group_to_perm_id", Integer(), nullable = False, unique = True, default = None, primary_key = True )
     user_id = Column( "user_id", Integer(), ForeignKey( 'users.user_id' ), nullable = False, unique = None, default = None )
     permission_id = Column( "permission_id", Integer(), ForeignKey( 'permissions.permission_id' ), nullable = False, unique = None, default = None )
     group_id = Column( "group_id", Integer(), ForeignKey( 'groups.group_id' ), nullable = False, unique = None, default = None )
 
     user = relationship( 'User' )
     permission = relationship( 'Permission' )
     group = relationship( 'Group' )
 
 class Statistics( Base, BaseModel ):
     __tablename__ = 'statistics'
     __table_args__ = ( UniqueConstraint( 'repository_id' ), {'extend_existing':True} )
     stat_id = Column( "stat_id", Integer(), nullable = False, unique = True, default = None, primary_key = True )
     repository_id = Column( "repository_id", Integer(), ForeignKey( 'repositories.repo_id' ), nullable = False, unique = True, default = None )
     stat_on_revision = Column( "stat_on_revision", Integer(), nullable = False )
     commit_activity = Column( "commit_activity", LargeBinary( 1000000 ), nullable = False )#JSON data
     commit_activity_combined = Column( "commit_activity_combined", LargeBinary(), nullable = False )#JSON data
     languages = Column( "languages", LargeBinary( 1000000 ), nullable = False )#JSON data
 
     repository = relationship( 'Repository', single_parent = True )
 
 class UserFollowing( Base, BaseModel ):
     __tablename__ = 'user_followings'
     __table_args__ = ( UniqueConstraint( 'user_id', 'follows_repository_id' ),
                       UniqueConstraint( 'user_id', 'follows_user_id' )
                       , {'extend_existing':True} )
 
     user_following_id = Column( "user_following_id", Integer(), nullable = False, unique = True, default = None, primary_key = True )
     user_id = Column( "user_id", Integer(), ForeignKey( 'users.user_id' ), nullable = False, unique = None, default = None )
     follows_repo_id = Column( "follows_repository_id", Integer(), ForeignKey( 'repositories.repo_id' ), nullable = True, unique = None, default = None )
     follows_user_id = Column( "follows_user_id", Integer(), ForeignKey( 'users.user_id' ), nullable = True, unique = None, default = None )
     follows_from = Column( 'follows_from', DateTime( timezone = False ), nullable = True, unique = None, default = datetime.datetime.now )
 
     user = relationship( 'User', primaryjoin = 'User.user_id==UserFollowing.user_id' )
 
     follows_user = relationship( 'User', primaryjoin = 'User.user_id==UserFollowing.follows_user_id' )
     follows_repository = relationship( 'Repository', order_by = 'Repository.repo_name' )
 
 
     @classmethod
     def get_repo_followers( cls, repo_id ):
         return cls.query().filter( cls.follows_repo_id == repo_id )
 
 class CacheInvalidation( Base, BaseModel ):
     __tablename__ = 'cache_invalidation'
     __table_args__ = ( UniqueConstraint( 'cache_key' ), {'extend_existing':True} )
     cache_id = Column( "cache_id", Integer(), nullable = False, unique = True, default = None, primary_key = True )
     cache_key = Column( "cache_key", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
     cache_args = Column( "cache_args", String( length = 255, convert_unicode = False, assert_unicode = None ), nullable = True, unique = None, default = None )
     cache_active = Column( "cache_active", Boolean(), nullable = True, unique = None, default = False )
 
 
     def __init__( self, cache_key, cache_args = '' ):
         self.cache_key = cache_key
         self.cache_args = cache_args
         self.cache_active = False
 
     def __repr__( self ):
         return "<%s('%s:%s')>" % ( self.__class__.__name__,
                                   self.cache_id, self.cache_key )
 
 class DbMigrateVersion( Base, BaseModel ):
     __tablename__ = 'db_migrate_version'
     __table_args__ = {'extend_existing':True}
     repository_id = Column( 'repository_id', String( 250 ), primary_key = True )
     repository_path = Column( 'repository_path', Text )
     version = Column( 'version', Integer )

rhodecode/model/forms.py

@@ -1,668 +1,681 b''
 """ this is forms validation classes
 http://formencode.org/module-formencode.validators.html
 for list off all availible validators
 
 we can create our own validators
 
 The table below outlines the options which can be used in a schema in addition to the validators themselves
 pre_validators          []     These validators will be applied before the schema
 chained_validators      []     These validators will be applied after the schema
 allow_extra_fields      False     If True, then it is not an error when keys that aren't associated with a validator are present
 filter_extra_fields     False     If True, then keys that aren't associated with a validator are removed
 if_key_missing          NoDefault If this is given, then any keys that aren't available but are expected will be replaced with this value (and then validated). This does not override a present .if_missing attribute on validators. NoDefault is a special FormEncode class to mean that no default values has been specified and therefore missing keys shouldn't take a default value.
 ignore_key_missing      False     If True, then missing keys will be missing in the result, if the validator doesn't have .if_missing on it already
 
 
 <name> = formencode.validators.<name of validator>
 <name> must equal form name
 list=[1,2,3,4,5]
 for SELECT use formencode.All(OneOf(list), Int())
 
 """
 import os
 import re
 import logging
 import traceback
 
 import formencode
 from formencode import All
 from formencode.validators import UnicodeString, OneOf, Int, Number, Regex, \
     Email, Bool, StringBoolean, Set
 
 from pylons.i18n.translation import _
 from webhelpers.pylonslib.secure_form import authentication_token
 
 from rhodecode.lib.utils import repo_name_slug
 from rhodecode.lib.auth import authenticate, get_crypt_password
 from rhodecode.lib.exceptions import LdapImportError
 from rhodecode.model.user import UserModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.db import User, UsersGroup, Group
 from rhodecode import BACKENDS
 
 log = logging.getLogger(__name__)
 
 #this is needed to translate the messages using _() in validators
 class State_obj(object):
     _ = staticmethod(_)
 
 #==============================================================================
 # VALIDATORS
 #==============================================================================
 class ValidAuthToken(formencode.validators.FancyValidator):
     messages = {'invalid_token':_('Token mismatch')}
 
     def validate_python(self, value, state):
 
         if value != authentication_token():
             raise formencode.Invalid(self.message('invalid_token', state,
                                             search_number=value), value, state)
 
 def ValidUsername(edit, old_data):
     class _ValidUsername(formencode.validators.FancyValidator):
 
         def validate_python(self, value, state):
             if value in ['default', 'new_user']:
                 raise formencode.Invalid(_('Invalid username'), value, state)
             #check if user is unique
             old_un = None
             if edit:
                 old_un = UserModel().get(old_data.get('user_id')).username
 
             if old_un != value or not edit:
                 if UserModel().get_by_username(value, cache=False,
                                                case_insensitive=True):
                     raise formencode.Invalid(_('This username already '
                                                'exists') , value, state)
 
             if re.match(r'^[a-zA-Z0-9]{1}[a-zA-Z0-9\-\_\.]+$', value) is None:
                 raise formencode.Invalid(_('Username may only contain '
                                            'alphanumeric characters '
                                            'underscores, periods or dashes '
                                            'and must begin with alphanumeric '
                                            'character'), value, state)
 
     return _ValidUsername
 
 
 def ValidUsersGroup(edit, old_data):
 
     class _ValidUsersGroup(formencode.validators.FancyValidator):
 
         def validate_python(self, value, state):
             if value in ['default']:
                 raise formencode.Invalid(_('Invalid group name'), value, state)
             #check if group is unique
             old_ugname = None
             if edit:
                 old_ugname = UsersGroup.get(
                             old_data.get('users_group_id')).users_group_name
 
             if old_ugname != value or not edit:
                 if UsersGroup.get_by_group_name(value, cache=False,
                                                case_insensitive=True):
                     raise formencode.Invalid(_('This users group '
                                                'already exists') , value,
                                              state)
 
 
             if re.match(r'^[a-zA-Z0-9]{1}[a-zA-Z0-9\-\_\.]+$', value) is None:
                 raise formencode.Invalid(_('Group name may only contain '
                                            'alphanumeric characters '
                                            'underscores, periods or dashes '
                                            'and must begin with alphanumeric '
                                            'character'), value, state)
 
     return _ValidUsersGroup
 
 
 def ValidReposGroup(edit, old_data):
     class _ValidReposGroup(formencode.validators.FancyValidator):
 
         def validate_python(self, value, state):
             #TODO WRITE VALIDATIONS
             group_name = value.get('group_name')
             group_parent_id = int(value.get('group_parent_id') or - 1)
 
             # slugify repo group just in case :)
             slug = repo_name_slug(group_name)
 
             # check for parent of self
             if edit and old_data['group_id'] == group_parent_id:
                     e_dict = {'group_parent_id':_('Cannot assign this group '
                                                   'as parent')}
                     raise formencode.Invalid('', value, state,
                                              error_dict=e_dict)
 
             old_gname = None
             if edit:
                 old_gname = Group.get(
                             old_data.get('group_id')).group_name
 
             if old_gname != group_name or not edit:
                 # check filesystem
                 gr = Group.query().filter(Group.group_name == slug)\
                     .filter(Group.group_parent_id == group_parent_id).scalar()
 
                 if gr:
                     e_dict = {'group_name':_('This group already exists')}
                     raise formencode.Invalid('', value, state,
                                              error_dict=e_dict)
 
     return _ValidReposGroup
 
 class ValidPassword(formencode.validators.FancyValidator):
 
     def to_python(self, value, state):
 
         if value:
 
             if value.get('password'):
                 try:
                     value['password'] = get_crypt_password(value['password'])
                 except UnicodeEncodeError:
                     e_dict = {'password':_('Invalid characters in password')}
                     raise formencode.Invalid('', value, state, error_dict=e_dict)
 
             if value.get('password_confirmation'):
                 try:
                     value['password_confirmation'] = \
                         get_crypt_password(value['password_confirmation'])
                 except UnicodeEncodeError:
                     e_dict = {'password_confirmation':_('Invalid characters in password')}
                     raise formencode.Invalid('', value, state, error_dict=e_dict)
 
             if value.get('new_password'):
                 try:
                     value['new_password'] = \
                         get_crypt_password(value['new_password'])
                 except UnicodeEncodeError:
                     e_dict = {'new_password':_('Invalid characters in password')}
                     raise formencode.Invalid('', value, state, error_dict=e_dict)
 
             return value
 
 class ValidPasswordsMatch(formencode.validators.FancyValidator):
 
     def validate_python(self, value, state):
 
         if value['password'] != value['password_confirmation']:
             e_dict = {'password_confirmation':
                    _('Passwords do not match')}
             raise formencode.Invalid('', value, state, error_dict=e_dict)
 
 class ValidAuth(formencode.validators.FancyValidator):
     messages = {
             'invalid_password':_('invalid password'),
             'invalid_login':_('invalid user name'),
             'disabled_account':_('Your account is disabled')
 
             }
     #error mapping
     e_dict = {'username':messages['invalid_login'],
               'password':messages['invalid_password']}
     e_dict_disable = {'username':messages['disabled_account']}
 
     def validate_python(self, value, state):
         password = value['password']
         username = value['username']
         user = UserModel().get_by_username(username)
 
         if authenticate(username, password):
             return value
         else:
             if user and user.active is False:
                 log.warning('user %s is disabled', username)
                 raise formencode.Invalid(self.message('disabled_account',
                                          state=State_obj),
                                          value, state,
                                          error_dict=self.e_dict_disable)
             else:
                 log.warning('user %s not authenticated', username)
                 raise formencode.Invalid(self.message('invalid_password',
                                          state=State_obj), value, state,
                                          error_dict=self.e_dict)
 
 class ValidRepoUser(formencode.validators.FancyValidator):
 
     def to_python(self, value, state):
         try:
             User.query().filter(User.active == True)\
                 .filter(User.username == value).one()
         except Exception:
             raise formencode.Invalid(_('This username is not valid'),
                                      value, state)
         return value
 
 def ValidRepoName(edit, old_data):
     class _ValidRepoName(formencode.validators.FancyValidator):
         def to_python(self, value, state):
 
             repo_name = value.get('repo_name')
 
             slug = repo_name_slug(repo_name)
             if slug in ['_admin', '']:
                 e_dict = {'repo_name': _('This repository name is disallowed')}
                 raise formencode.Invalid('', value, state, error_dict=e_dict)
 
 
             if value.get('repo_group'):
                 gr = Group.get(value.get('repo_group'))
                 group_path = gr.full_path
                 # value needs to be aware of group name in order to check
                 # db key This is an actuall just the name to store in the
                 # database
                 repo_name_full = group_path + Group.url_sep() + repo_name
             else:
                 group_path = ''
                 repo_name_full = repo_name
 
 
             value['repo_name_full'] = repo_name_full
             if old_data.get('repo_name') != repo_name_full or not edit:
 
                 if group_path != '':
                     if RepoModel().get_by_repo_name(repo_name_full,):
                         e_dict = {'repo_name':_('This repository already '
                                                 'exists in group "%s"') %
                                   gr.group_name}
                         raise formencode.Invalid('', value, state,
                                                  error_dict=e_dict)
 
                 else:
                     if RepoModel().get_by_repo_name(repo_name_full):
                         e_dict = {'repo_name':_('This repository '
                                                 'already exists')}
                         raise formencode.Invalid('', value, state,
                                                  error_dict=e_dict)
             return value
 
 
     return _ValidRepoName
 
 def ValidForkName():
     class _ValidForkName(formencode.validators.FancyValidator):
         def to_python(self, value, state):
+
+            repo_name = value.get('fork_name')
+
+            slug = repo_name_slug(repo_name)
+            if slug in ['_admin', '']:
+                e_dict = {'repo_name': _('This repository name is disallowed')}
+                raise formencode.Invalid('', value, state, error_dict=e_dict)
+
+            if RepoModel().get_by_repo_name(repo_name):
+                e_dict = {'fork_name':_('This repository '
+                                        'already exists')}
+                raise formencode.Invalid('', value, state,
+                                         error_dict=e_dict)
             return value
     return _ValidForkName
 
 
 def SlugifyName():
     class _SlugifyName(formencode.validators.FancyValidator):
 
         def to_python(self, value, state):
             return repo_name_slug(value)
 
     return _SlugifyName
 
 def ValidCloneUri():
     from mercurial.httprepo import httprepository, httpsrepository
     from rhodecode.lib.utils import make_ui
 
     class _ValidCloneUri(formencode.validators.FancyValidator):
 
         def to_python(self, value, state):
             if not value:
                 pass
             elif value.startswith('https'):
                 try:
                     httpsrepository(make_ui('db'), value).capabilities
                 except Exception, e:
                     log.error(traceback.format_exc())
                     raise formencode.Invalid(_('invalid clone url'), value,
                                              state)
             elif value.startswith('http'):
                 try:
                     httprepository(make_ui('db'), value).capabilities
                 except Exception, e:
                     log.error(traceback.format_exc())
                     raise formencode.Invalid(_('invalid clone url'), value,
                                              state)
             else:
                 raise formencode.Invalid(_('Invalid clone url, provide a '
                                            'valid clone http\s url'), value,
                                          state)
             return value
 
     return _ValidCloneUri
 
 def ValidForkType(old_data):
     class _ValidForkType(formencode.validators.FancyValidator):
 
         def to_python(self, value, state):
             if old_data['repo_type'] != value:
                 raise formencode.Invalid(_('Fork have to be the same '
                                            'type as original'), value, state)
 
             return value
     return _ValidForkType
 
 class ValidPerms(formencode.validators.FancyValidator):
     messages = {'perm_new_member_name':_('This username or users group name'
                                          ' is not valid')}
 
     def to_python(self, value, state):
         perms_update = []
         perms_new = []
         #build a list of permission to update and new permission to create
         for k, v in value.items():
             #means new added member to permissions
             if k.startswith('perm_new_member'):
                 new_perm = value.get('perm_new_member', False)
                 new_member = value.get('perm_new_member_name', False)
                 new_type = value.get('perm_new_member_type')
 
                 if new_member and new_perm:
                     if (new_member, new_perm, new_type) not in perms_new:
                         perms_new.append((new_member, new_perm, new_type))
             elif k.startswith('u_perm_') or k.startswith('g_perm_'):
                 member = k[7:]
                 t = {'u':'user',
                      'g':'users_group'}[k[0]]
                 if member == 'default':
                     if value['private']:
                         #set none for default when updating to private repo
                         v = 'repository.none'
                 perms_update.append((member, v, t))
 
         value['perms_updates'] = perms_update
         value['perms_new'] = perms_new
 
         #update permissions
         for k, v, t in perms_new:
             try:
                 if t is 'user':
                     self.user_db = User.query()\
                         .filter(User.active == True)\
                         .filter(User.username == k).one()
                 if t is 'users_group':
                     self.user_db = UsersGroup.query()\
                         .filter(UsersGroup.users_group_active == True)\
                         .filter(UsersGroup.users_group_name == k).one()
 
             except Exception:
                 msg = self.message('perm_new_member_name',
                                      state=State_obj)
                 raise formencode.Invalid(msg, value, state,
                                          error_dict={'perm_new_member_name':msg})
         return value
 
 class ValidSettings(formencode.validators.FancyValidator):
 
     def to_python(self, value, state):
         #settings  form can't edit user
         if value.has_key('user'):
             del['value']['user']
 
         return value
 
 class ValidPath(formencode.validators.FancyValidator):
     def to_python(self, value, state):
 
         if not os.path.isdir(value):
             msg = _('This is not a valid path')
             raise formencode.Invalid(msg, value, state,
                                      error_dict={'paths_root_path':msg})
         return value
 
 def UniqSystemEmail(old_data):
     class _UniqSystemEmail(formencode.validators.FancyValidator):
         def to_python(self, value, state):
             value = value.lower()
             if old_data.get('email') != value:
                 user = User.query().filter(User.email == value).scalar()
                 if user:
                     raise formencode.Invalid(
                                     _("This e-mail address is already taken"),
                                     value, state)
             return value
 
     return _UniqSystemEmail
 
 class ValidSystemEmail(formencode.validators.FancyValidator):
     def to_python(self, value, state):
         value = value.lower()
         user = User.query().filter(User.email == value).scalar()
         if  user is None:
             raise formencode.Invalid(_("This e-mail address doesn't exist.") ,
                                      value, state)
 
         return value
 
 class LdapLibValidator(formencode.validators.FancyValidator):
 
     def to_python(self, value, state):
 
         try:
             import ldap
         except ImportError:
             raise LdapImportError
         return value
 
 class AttrLoginValidator(formencode.validators.FancyValidator):
 
     def to_python(self, value, state):
 
         if not value or not isinstance(value, (str, unicode)):
             raise formencode.Invalid(_("The LDAP Login attribute of the CN "
                                        "must be specified - this is the name "
                                        "of the attribute that is equivalent "
                                        "to 'username'"),
                                      value, state)
 
         return value
 
 #===============================================================================
 # FORMS
 #===============================================================================
 class LoginForm(formencode.Schema):
     allow_extra_fields = True
     filter_extra_fields = True
     username = UnicodeString(
                              strip=True,
                              min=1,
                              not_empty=True,
                              messages={
                                 'empty':_('Please enter a login'),
                                 'tooShort':_('Enter a value %(min)i characters long or more')}
                             )
 
     password = UnicodeString(
                             strip=True,
                             min=3,
                             not_empty=True,
                             messages={
                                 'empty':_('Please enter a password'),
                                 'tooShort':_('Enter %(min)i characters or more')}
                                 )
 
 
     #chained validators have access to all data
     chained_validators = [ValidAuth]
 
 def UserForm(edit=False, old_data={}):
     class _UserForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         username = All(UnicodeString(strip=True, min=1, not_empty=True),
                        ValidUsername(edit, old_data))
         if edit:
             new_password = All(UnicodeString(strip=True, min=6, not_empty=False))
             admin = StringBoolean(if_missing=False)
         else:
             password = All(UnicodeString(strip=True, min=6, not_empty=True))
         active = StringBoolean(if_missing=False)
         name = UnicodeString(strip=True, min=1, not_empty=True)
         lastname = UnicodeString(strip=True, min=1, not_empty=True)
         email = All(Email(not_empty=True), UniqSystemEmail(old_data))
 
         chained_validators = [ValidPassword]
 
     return _UserForm
 
 
 def UsersGroupForm(edit=False, old_data={}, available_members=[]):
     class _UsersGroupForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
 
         users_group_name = All(UnicodeString(strip=True, min=1, not_empty=True),
                        ValidUsersGroup(edit, old_data))
 
         users_group_active = StringBoolean(if_missing=False)
 
         if edit:
             users_group_members = OneOf(available_members, hideList=False,
                                         testValueList=True,
                                         if_missing=None, not_empty=False)
 
     return _UsersGroupForm
 
 def ReposGroupForm(edit=False, old_data={}, available_groups=[]):
     class _ReposGroupForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
 
         group_name = All(UnicodeString(strip=True, min=1, not_empty=True),
                                SlugifyName())
         group_description = UnicodeString(strip=True, min=1,
                                                 not_empty=True)
         group_parent_id = OneOf(available_groups, hideList=False,
                                         testValueList=True,
                                         if_missing=None, not_empty=False)
 
         chained_validators = [ValidReposGroup(edit, old_data)]
 
     return _ReposGroupForm
 
 def RegisterForm(edit=False, old_data={}):
     class _RegisterForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         username = All(ValidUsername(edit, old_data),
                        UnicodeString(strip=True, min=1, not_empty=True))
         password = All(UnicodeString(strip=True, min=6, not_empty=True))
         password_confirmation = All(UnicodeString(strip=True, min=6, not_empty=True))
         active = StringBoolean(if_missing=False)
         name = UnicodeString(strip=True, min=1, not_empty=True)
         lastname = UnicodeString(strip=True, min=1, not_empty=True)
         email = All(Email(not_empty=True), UniqSystemEmail(old_data))
 
         chained_validators = [ValidPasswordsMatch, ValidPassword]
 
     return _RegisterForm
 
 def PasswordResetForm():
     class _PasswordResetForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         email = All(ValidSystemEmail(), Email(not_empty=True))
     return _PasswordResetForm
 
 def RepoForm(edit=False, old_data={}, supported_backends=BACKENDS.keys(),
              repo_groups=[]):
     class _RepoForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         repo_name = All(UnicodeString(strip=True, min=1, not_empty=True),
                         SlugifyName())
         clone_uri = All(UnicodeString(strip=True, min=1, not_empty=False),
                         ValidCloneUri()())
         repo_group = OneOf(repo_groups, hideList=True)
         repo_type = OneOf(supported_backends)
         description = UnicodeString(strip=True, min=1, not_empty=True)
         private = StringBoolean(if_missing=False)
         enable_statistics = StringBoolean(if_missing=False)
         enable_downloads = StringBoolean(if_missing=False)
 
         if edit:
             #this is repo owner
             user = All(UnicodeString(not_empty=True), ValidRepoUser)
 
         chained_validators = [ValidRepoName(edit, old_data), ValidPerms]
     return _RepoForm
 
 def RepoForkForm(edit=False, old_data={}, supported_backends=BACKENDS.keys()):
     class _RepoForkForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         fork_name = All(UnicodeString(strip=True, min=1, not_empty=True),
                         SlugifyName())
         description = UnicodeString(strip=True, min=1, not_empty=True)
         private = StringBoolean(if_missing=False)
         repo_type = All(ValidForkType(old_data), OneOf(supported_backends))
 
         chained_validators = [ValidForkName()]
 
     return _RepoForkForm
 
 def RepoSettingsForm(edit=False, old_data={}):
     class _RepoForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         repo_name = All(UnicodeString(strip=True, min=1, not_empty=True),
                         SlugifyName())
         description = UnicodeString(strip=True, min=1, not_empty=True)
         private = StringBoolean(if_missing=False)
 
         chained_validators = [ValidRepoName(edit, old_data), ValidPerms, ValidSettings]
     return _RepoForm
 
 
 def ApplicationSettingsForm():
     class _ApplicationSettingsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         rhodecode_title = UnicodeString(strip=True, min=1, not_empty=True)
         rhodecode_realm = UnicodeString(strip=True, min=1, not_empty=True)
         rhodecode_ga_code = UnicodeString(strip=True, min=1, not_empty=False)
 
     return _ApplicationSettingsForm
 
 def ApplicationUiSettingsForm():
     class _ApplicationUiSettingsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         web_push_ssl = OneOf(['true', 'false'], if_missing='false')
         paths_root_path = All(ValidPath(), UnicodeString(strip=True, min=1, not_empty=True))
         hooks_changegroup_update = OneOf(['True', 'False'], if_missing=False)
         hooks_changegroup_repo_size = OneOf(['True', 'False'], if_missing=False)
         hooks_pretxnchangegroup_push_logger = OneOf(['True', 'False'], if_missing=False)
         hooks_preoutgoing_pull_logger = OneOf(['True', 'False'], if_missing=False)
 
     return _ApplicationUiSettingsForm
 
 def DefaultPermissionsForm(perms_choices, register_choices, create_choices):
     class _DefaultPermissionsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         overwrite_default = StringBoolean(if_missing=False)
         anonymous = OneOf(['True', 'False'], if_missing=False)
         default_perm = OneOf(perms_choices)
         default_register = OneOf(register_choices)
         default_create = OneOf(create_choices)
 
     return _DefaultPermissionsForm
 
 
 def LdapSettingsForm(tls_reqcert_choices, search_scope_choices, tls_kind_choices):
     class _LdapSettingsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         pre_validators = [LdapLibValidator]
         ldap_active = StringBoolean(if_missing=False)
         ldap_host = UnicodeString(strip=True,)
         ldap_port = Number(strip=True,)
         ldap_tls_kind = OneOf(tls_kind_choices)
         ldap_tls_reqcert = OneOf(tls_reqcert_choices)
         ldap_dn_user = UnicodeString(strip=True,)
         ldap_dn_pass = UnicodeString(strip=True,)
         ldap_base_dn = UnicodeString(strip=True,)
         ldap_filter = UnicodeString(strip=True,)
         ldap_search_scope = OneOf(search_scope_choices)
         ldap_attr_login = All(AttrLoginValidator, UnicodeString(strip=True,))
         ldap_attr_firstname = UnicodeString(strip=True,)
         ldap_attr_lastname = UnicodeString(strip=True,)
         ldap_attr_email = UnicodeString(strip=True,)
 
     return _LdapSettingsForm

rhodecode/model/repo.py

@@ -1,358 +1,361 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.model.repo
     ~~~~~~~~~~~~~~~~~~~~
 
     Repository model for rhodecode
 
     :created_on: Jun 5, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import os
 import shutil
 import logging
 import traceback
 from datetime import datetime
 
 from sqlalchemy.orm import joinedload, make_transient
 
 from vcs.utils.lazy import LazyProperty
 from vcs.backends import get_backend
 
 from rhodecode.lib import safe_str
 
 from rhodecode.model import BaseModel
 from rhodecode.model.caching_query import FromCache
 from rhodecode.model.db import Repository, RepoToPerm, User, Permission, \
     Statistics, UsersGroup, UsersGroupRepoToPerm, RhodeCodeUi, Group
 from rhodecode.model.user import UserModel
 
 log = logging.getLogger(__name__)
 
 
 class RepoModel(BaseModel):
 
     @LazyProperty
     def repos_path(self):
         """Get's the repositories root path from database
         """
 
         q = self.sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
         return q.ui_value
 
     def get(self, repo_id, cache=False):
         repo = self.sa.query(Repository)\
             .filter(Repository.repo_id == repo_id)
 
         if cache:
             repo = repo.options(FromCache("sql_cache_short",
                                           "get_repo_%s" % repo_id))
         return repo.scalar()
 
     def get_by_repo_name(self, repo_name, cache=False):
         repo = self.sa.query(Repository)\
             .filter(Repository.repo_name == repo_name)
 
         if cache:
             repo = repo.options(FromCache("sql_cache_short",
                                           "get_repo_%s" % repo_name))
         return repo.scalar()
 
 
     def get_users_js(self):
 
         users = self.sa.query(User).filter(User.active == True).all()
         u_tmpl = '''{id:%s, fname:"%s", lname:"%s", nname:"%s"},'''
         users_array = '[%s]' % '\n'.join([u_tmpl % (u.user_id, u.name,
                                                     u.lastname, u.username)
                                         for u in users])
         return users_array
 
     def get_users_groups_js(self):
         users_groups = self.sa.query(UsersGroup)\
             .filter(UsersGroup.users_group_active == True).all()
 
         g_tmpl = '''{id:%s, grname:"%s",grmembers:"%s"},'''
 
         users_groups_array = '[%s]' % '\n'.join([g_tmpl % \
                                     (gr.users_group_id, gr.users_group_name,
                                      len(gr.members))
                                         for gr in users_groups])
         return users_groups_array
 
     def update(self, repo_name, form_data):
         try:
             cur_repo = self.get_by_repo_name(repo_name, cache=False)
 
             #update permissions
             for member, perm, member_type in form_data['perms_updates']:
                 if member_type == 'user':
                     r2p = self.sa.query(RepoToPerm)\
                             .filter(RepoToPerm.user == User.by_username(member))\
                             .filter(RepoToPerm.repository == cur_repo)\
                             .one()
 
                     r2p.permission = self.sa.query(Permission)\
                                         .filter(Permission.permission_name ==
                                                 perm).scalar()
                     self.sa.add(r2p)
                 else:
                     g2p = self.sa.query(UsersGroupRepoToPerm)\
                             .filter(UsersGroupRepoToPerm.users_group ==
                                     UsersGroup.get_by_group_name(member))\
                             .filter(UsersGroupRepoToPerm.repository ==
                                     cur_repo).one()
 
                     g2p.permission = self.sa.query(Permission)\
                                         .filter(Permission.permission_name ==
                                                 perm).scalar()
                     self.sa.add(g2p)
 
             #set new permissions
             for member, perm, member_type in form_data['perms_new']:
                 if member_type == 'user':
                     r2p = RepoToPerm()
                     r2p.repository = cur_repo
                     r2p.user = User.by_username(member)
 
                     r2p.permission = self.sa.query(Permission)\
                                         .filter(Permission.
                                                 permission_name == perm)\
                                                 .scalar()
                     self.sa.add(r2p)
                 else:
                     g2p = UsersGroupRepoToPerm()
                     g2p.repository = cur_repo
                     g2p.users_group = UsersGroup.get_by_group_name(member)
                     g2p.permission = self.sa.query(Permission)\
                                         .filter(Permission.
                                                 permission_name == perm)\
                                                 .scalar()
                     self.sa.add(g2p)
 
             #update current repo
             for k, v in form_data.items():
                 if k == 'user':
                     cur_repo.user = User.by_username(v)
                 elif k == 'repo_name':
                     cur_repo.repo_name = form_data['repo_name_full']
                 elif k == 'repo_group':
                     cur_repo.group_id = v
 
                 else:
                     setattr(cur_repo, k, v)
 
             self.sa.add(cur_repo)
 
             if repo_name != form_data['repo_name_full']:
                 # rename repository
                 self.__rename_repo(old=repo_name,
                                    new=form_data['repo_name_full'])
 
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def create(self, form_data, cur_user, just_db=False, fork=False):
 
         try:
             if fork:
                 repo_name = form_data['fork_name']
                 org_name = form_data['repo_name']
                 org_full_name = org_name
 
             else:
                 org_name = repo_name = form_data['repo_name']
                 repo_name_full = form_data['repo_name_full']
 
             new_repo = Repository()
             new_repo.enable_statistics = False
             for k, v in form_data.items():
                 if k == 'repo_name':
                     if fork:
                         v = repo_name
                     else:
                         v = repo_name_full
                 if k == 'repo_group':
                     k = 'group_id'
 
+                if k == 'description':
+                    v = v or repo_name
+
                 setattr(new_repo, k, v)
 
             if fork:
                 parent_repo = self.sa.query(Repository)\
                         .filter(Repository.repo_name == org_full_name).one()
                 new_repo.fork = parent_repo
 
             new_repo.user_id = cur_user.user_id
             self.sa.add(new_repo)
 
             #create default permission
             repo_to_perm = RepoToPerm()
             default = 'repository.read'
             for p in UserModel(self.sa).get_by_username('default',
                                                     cache=False).user_perms:
                 if p.permission.permission_name.startswith('repository.'):
                     default = p.permission.permission_name
                     break
 
             default_perm = 'repository.none' if form_data['private'] else default
 
             repo_to_perm.permission_id = self.sa.query(Permission)\
                     .filter(Permission.permission_name == default_perm)\
                     .one().permission_id
 
             repo_to_perm.repository = new_repo
             repo_to_perm.user_id = UserModel(self.sa)\
                 .get_by_username('default', cache=False).user_id
 
             self.sa.add(repo_to_perm)
 
             if not just_db:
                 self.__create_repo(repo_name, form_data['repo_type'],
                                    form_data['repo_group'],
                                    form_data['clone_uri'])
 
             self.sa.commit()
 
             #now automatically start following this repository as owner
             from rhodecode.model.scm import ScmModel
             ScmModel(self.sa).toggle_following_repo(new_repo.repo_id,
                                              cur_user.user_id)
 
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def create_fork(self, form_data, cur_user):
         from rhodecode.lib.celerylib import tasks, run_task
         run_task(tasks.create_repo_fork, form_data, cur_user)
 
     def delete(self, repo):
         try:
             self.sa.delete(repo)
             self.__delete_repo(repo)
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def delete_perm_user(self, form_data, repo_name):
         try:
             self.sa.query(RepoToPerm)\
                 .filter(RepoToPerm.repository \
                         == self.get_by_repo_name(repo_name))\
                 .filter(RepoToPerm.user_id == form_data['user_id']).delete()
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def delete_perm_users_group(self, form_data, repo_name):
         try:
             self.sa.query(UsersGroupRepoToPerm)\
                 .filter(UsersGroupRepoToPerm.repository \
                         == self.get_by_repo_name(repo_name))\
                 .filter(UsersGroupRepoToPerm.users_group_id \
                         == form_data['users_group_id']).delete()
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def delete_stats(self, repo_name):
         try:
             self.sa.query(Statistics)\
                 .filter(Statistics.repository == \
                         self.get_by_repo_name(repo_name)).delete()
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def __create_repo(self, repo_name, alias, new_parent_id, clone_uri=False):
         """
         makes repository on filesystem. It's group aware means it'll create
         a repository within a group, and alter the paths accordingly of
         group location
 
         :param repo_name:
         :param alias:
         :param parent_id:
         :param clone_uri:
         """
         from rhodecode.lib.utils import is_valid_repo
-        
+
         if new_parent_id:
             paths = Group.get(new_parent_id).full_path.split(Group.url_sep())
             new_parent_path = os.sep.join(paths)
         else:
             new_parent_path = ''
 
         repo_path = os.path.join(*map(lambda x:safe_str(x),
                                 [self.repos_path, new_parent_path, repo_name]))
 
         if is_valid_repo(repo_path, self.repos_path) is False:
             log.info('creating repo %s in %s @ %s', repo_name, repo_path,
                      clone_uri)
             backend = get_backend(alias)
 
             backend(repo_path, create=True, src_url=clone_uri)
 
 
     def __rename_repo(self, old, new):
         """
         renames repository on filesystem
 
         :param old: old name
         :param new: new name
         """
         log.info('renaming repo from %s to %s', old, new)
 
         old_path = os.path.join(self.repos_path, old)
         new_path = os.path.join(self.repos_path, new)
         if os.path.isdir(new_path):
             raise Exception('Was trying to rename to already existing dir %s' \
             		     % new_path)
         shutil.move(old_path, new_path)
 
     def __delete_repo(self, repo):
         """
         removes repo from filesystem, the removal is acctually made by
         added rm__ prefix into dir, and rename internat .hg/.git dirs so this
         repository is no longer valid for rhodecode, can be undeleted later on
         by reverting the renames on this repository
 
         :param repo: repo object
         """
         rm_path = os.path.join(self.repos_path, repo.repo_name)
         log.info("Removing %s", rm_path)
         #disable hg/git
         alias = repo.repo_type
         shutil.move(os.path.join(rm_path, '.%s' % alias),
                     os.path.join(rm_path, 'rm__.%s' % alias))
         #disable repo
         shutil.move(rm_path, os.path.join(self.repos_path, 'rm__%s__%s' \
                                           % (datetime.today()\
                                              .strftime('%Y%m%d_%H%M%S_%f'),
                                             repo.repo_name)))

rhodecode/model/user.py

@@ -1,387 +1,388 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.model.user
     ~~~~~~~~~~~~~~~~~~~~
 
     users model for RhodeCode
 
     :created_on: Apr 9, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
 import traceback
 
 from pylons.i18n.translation import _
 
+from rhodecode.lib import safe_unicode
 from rhodecode.model import BaseModel
 from rhodecode.model.caching_query import FromCache
 from rhodecode.model.db import User, RepoToPerm, Repository, Permission, \
     UserToPerm, UsersGroupRepoToPerm, UsersGroupToPerm, UsersGroupMember
 from rhodecode.lib.exceptions import DefaultUserException, \
     UserOwnsReposException
 
 from sqlalchemy.exc import DatabaseError
 from rhodecode.lib import generate_api_key
 from sqlalchemy.orm import joinedload
 
 log = logging.getLogger(__name__)
 
 PERM_WEIGHTS = {'repository.none': 0,
                 'repository.read': 1,
                 'repository.write': 3,
                 'repository.admin': 3}
 
 
 class UserModel(BaseModel):
     def get(self, user_id, cache=False):
         user = self.sa.query(User)
         if cache:
             user = user.options(FromCache("sql_cache_short",
                                           "get_user_%s" % user_id))
         return user.get(user_id)
 
     def get_by_username(self, username, cache=False, case_insensitive=False):
 
         if case_insensitive:
             user = self.sa.query(User).filter(User.username.ilike(username))
         else:
             user = self.sa.query(User)\
                 .filter(User.username == username)
         if cache:
             user = user.options(FromCache("sql_cache_short",
                                           "get_user_%s" % username))
         return user.scalar()
 
     def get_by_api_key(self, api_key, cache=False):
 
         user = self.sa.query(User)\
                 .filter(User.api_key == api_key)
         if cache:
             user = user.options(FromCache("sql_cache_short",
                                           "get_user_%s" % api_key))
         return user.scalar()
 
     def create(self, form_data):
         try:
             new_user = User()
             for k, v in form_data.items():
                 setattr(new_user, k, v)
 
             new_user.api_key = generate_api_key(form_data['username'])
             self.sa.add(new_user)
             self.sa.commit()
             return new_user
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def create_ldap(self, username, password, user_dn, attrs):
         """
         Checks if user is in database, if not creates this user marked
         as ldap user
         :param username:
         :param password:
         :param user_dn:
         :param attrs:
         """
         from rhodecode.lib.auth import get_crypt_password
         log.debug('Checking for such ldap account in RhodeCode database')
         if self.get_by_username(username, case_insensitive=True) is None:
             try:
                 new_user = User()
                 # add ldap account always lowercase
                 new_user.username = username.lower()
                 new_user.password = get_crypt_password(password)
                 new_user.api_key = generate_api_key(username)
                 new_user.email = attrs['email']
                 new_user.active = True
-                new_user.ldap_dn = user_dn
+                new_user.ldap_dn = safe_unicode(user_dn)
                 new_user.name = attrs['name']
                 new_user.lastname = attrs['lastname']
 
                 self.sa.add(new_user)
                 self.sa.commit()
                 return True
             except (DatabaseError,):
                 log.error(traceback.format_exc())
                 self.sa.rollback()
                 raise
         log.debug('this %s user exists skipping creation of ldap account',
                   username)
         return False
 
     def create_registration(self, form_data):
         from rhodecode.lib.celerylib import tasks, run_task
         try:
             new_user = User()
             for k, v in form_data.items():
                 if k != 'admin':
                     setattr(new_user, k, v)
 
             self.sa.add(new_user)
             self.sa.commit()
             body = ('New user registration\n'
                     'username: %s\n'
                     'email: %s\n')
             body = body % (form_data['username'], form_data['email'])
 
             run_task(tasks.send_email, None,
                      _('[RhodeCode] New User registration'),
                      body)
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def update(self, user_id, form_data):
         try:
             user = self.get(user_id, cache=False)
             if user.username == 'default':
                 raise DefaultUserException(
                                 _("You can't Edit this user since it's"
                                   " crucial for entire application"))
 
             for k, v in form_data.items():
                 if k == 'new_password' and v != '':
                     user.password = v
                     user.api_key = generate_api_key(user.username)
                 else:
                     setattr(user, k, v)
 
             self.sa.add(user)
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def update_my_account(self, user_id, form_data):
         try:
             user = self.get(user_id, cache=False)
             if user.username == 'default':
                 raise DefaultUserException(
                                 _("You can't Edit this user since it's"
                                   " crucial for entire application"))
             for k, v in form_data.items():
                 if k == 'new_password' and v != '':
                     user.password = v
                     user.api_key = generate_api_key(user.username)
                 else:
                     if k not in ['admin', 'active']:
                         setattr(user, k, v)
 
             self.sa.add(user)
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def delete(self, user_id):
         try:
             user = self.get(user_id, cache=False)
             if user.username == 'default':
                 raise DefaultUserException(
                                 _("You can't remove this user since it's"
                                   " crucial for entire application"))
             if user.repositories:
                 raise UserOwnsReposException(_('This user still owns %s '
                                                'repositories and cannot be '
                                                'removed. Switch owners or '
                                                'remove those repositories') \
                                                % user.repositories)
             self.sa.delete(user)
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def reset_password_link(self, data):
         from rhodecode.lib.celerylib import tasks, run_task
         run_task(tasks.send_password_link, data['email'])
 
     def reset_password(self, data):
         from rhodecode.lib.celerylib import tasks, run_task
         run_task(tasks.reset_user_password, data['email'])
 
     def fill_data(self, auth_user, user_id=None, api_key=None):
         """
         Fetches auth_user by user_id,or api_key if present.
         Fills auth_user attributes with those taken from database.
         Additionally set's is_authenitated if lookup fails
         present in database
 
         :param auth_user: instance of user to set attributes
         :param user_id: user id to fetch by
         :param api_key: api key to fetch by
         """
         if user_id is None and api_key is None:
             raise Exception('You need to pass user_id or api_key')
 
         try:
             if api_key:
                 dbuser = self.get_by_api_key(api_key)
             else:
                 dbuser = self.get(user_id)
 
             if dbuser is not None:
                 log.debug('filling %s data', dbuser)
                 for k, v in dbuser.get_dict().items():
                     setattr(auth_user, k, v)
 
         except:
             log.error(traceback.format_exc())
             auth_user.is_authenticated = False
 
         return auth_user
 
     def fill_perms(self, user):
         """
         Fills user permission attribute with permissions taken from database
         works for permissions given for repositories, and for permissions that
         are granted to groups
 
         :param user: user instance to fill his perms
         """
 
         user.permissions['repositories'] = {}
         user.permissions['global'] = set()
 
         #======================================================================
         # fetch default permissions
         #======================================================================
         default_user = self.get_by_username('default', cache=True)
 
         default_perms = self.sa.query(RepoToPerm, Repository, Permission)\
             .join((Repository, RepoToPerm.repository_id ==
                    Repository.repo_id))\
             .join((Permission, RepoToPerm.permission_id ==
                    Permission.permission_id))\
             .filter(RepoToPerm.user == default_user).all()
 
         if user.is_admin:
             #==================================================================
             # #admin have all default rights set to admin
             #==================================================================
             user.permissions['global'].add('hg.admin')
 
             for perm in default_perms:
                 p = 'repository.admin'
                 user.permissions['repositories'][perm.RepoToPerm.
                                                  repository.repo_name] = p
 
         else:
             #==================================================================
             # set default permissions
             #==================================================================
             uid = user.user_id
 
             #default global
             default_global_perms = self.sa.query(UserToPerm)\
                 .filter(UserToPerm.user == default_user)
 
             for perm in default_global_perms:
                 user.permissions['global'].add(perm.permission.permission_name)
 
             #default for repositories
             for perm in default_perms:
                 if perm.Repository.private and not (perm.Repository.user_id ==
                                                     uid):
                     #diself.sable defaults for private repos,
                     p = 'repository.none'
                 elif perm.Repository.user_id == uid:
                     #set admin if owner
                     p = 'repository.admin'
                 else:
                     p = perm.Permission.permission_name
 
                 user.permissions['repositories'][perm.RepoToPerm.
                                                  repository.repo_name] = p
 
             #==================================================================
             # overwrite default with user permissions if any
             #==================================================================
 
             #user global
             user_perms = self.sa.query(UserToPerm)\
                     .options(joinedload(UserToPerm.permission))\
                     .filter(UserToPerm.user_id == uid).all()
 
             for perm in user_perms:
                 user.permissions['global'].add(perm.permission.
                                                permission_name)
 
             #user repositories
             user_repo_perms = self.sa.query(RepoToPerm, Permission,
                                             Repository)\
                 .join((Repository, RepoToPerm.repository_id ==
                        Repository.repo_id))\
                 .join((Permission, RepoToPerm.permission_id ==
                        Permission.permission_id))\
                 .filter(RepoToPerm.user_id == uid).all()
 
             for perm in user_repo_perms:
                 # set admin if owner
                 if perm.Repository.user_id == uid:
                     p = 'repository.admin'
                 else:
                     p = perm.Permission.permission_name
                 user.permissions['repositories'][perm.RepoToPerm.
                                                  repository.repo_name] = p
 
             #==================================================================
             # check if user is part of groups for this repository and fill in
             # (or replace with higher) permissions
             #==================================================================
 
             #users group global
             user_perms_from_users_groups = self.sa.query(UsersGroupToPerm)\
                 .options(joinedload(UsersGroupToPerm.permission))\
                 .join((UsersGroupMember, UsersGroupToPerm.users_group_id ==
                        UsersGroupMember.users_group_id))\
                 .filter(UsersGroupMember.user_id == uid).all()
 
             for perm in user_perms_from_users_groups:
                 user.permissions['global'].add(perm.permission.permission_name)
 
             #users group repositories
             user_repo_perms_from_users_groups = self.sa.query(
                                                 UsersGroupRepoToPerm,
                                                 Permission, Repository,)\
                 .join((Repository, UsersGroupRepoToPerm.repository_id ==
                        Repository.repo_id))\
                 .join((Permission, UsersGroupRepoToPerm.permission_id ==
                        Permission.permission_id))\
                 .join((UsersGroupMember, UsersGroupRepoToPerm.users_group_id ==
                        UsersGroupMember.users_group_id))\
                 .filter(UsersGroupMember.user_id == uid).all()
 
             for perm in user_repo_perms_from_users_groups:
                 p = perm.Permission.permission_name
                 cur_perm = user.permissions['repositories'][perm.
                                                     UsersGroupRepoToPerm.
                                                     repository.repo_name]
                 #overwrite permission only if it's greater than permission
                 # given from other sources
                 if PERM_WEIGHTS[p] > PERM_WEIGHTS[cur_perm]:
                     user.permissions['repositories'][perm.UsersGroupRepoToPerm.
                                                      repository.repo_name] = p
 
         return user

rhodecode/tests/rhodecode_crawler.py

@@ -1,148 +1,148 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.tests.test_crawer
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Test for crawling a project for memory usage
     This should be runned just as regular script together
     with a watch script that will show memory usage.
     
     watch -n1 ./rhodecode/tests/mem_watch
 
     :created_on: Apr 21, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
 import cookielib
 import urllib
 import urllib2
 import vcs
 import time
 
 from os.path import join as jn
 
 
 BASE_URI = 'http://127.0.0.1:5000/%s'
 PROJECT = 'CPython'
 PROJECT_PATH = jn('/', 'home', 'marcink', 'hg_repos')
 
 
 cj = cookielib.FileCookieJar('/tmp/rc_test_cookie.txt')
 o = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
 o.addheaders = [
                      ('User-agent', 'rhodecode-crawler'),
                      ('Accept-Language', 'en - us, en;q = 0.5')
                     ]
 
 urllib2.install_opener(o)
 
 
 
 def test_changelog_walk(pages=100):
     total_time = 0
     for i in range(1, pages):
 
         page = '/'.join((PROJECT, 'changelog',))
 
         full_uri = (BASE_URI % page) + '?' + urllib.urlencode({'page':i})
         s = time.time()
         f = o.open(full_uri)
         size = len(f.read())
         e = time.time() - s
         total_time += e
         print 'visited %s size:%s req:%s ms' % (full_uri, size, e)
 
 
     print 'total_time', total_time
     print 'average on req', total_time / float(pages)
 
 
 def test_changeset_walk(limit=None):
     print 'processing', jn(PROJECT_PATH, PROJECT)
     total_time = 0
 
     repo = vcs.get_repo(jn(PROJECT_PATH, PROJECT))
     cnt = 0
     for i in repo:
         cnt += 1
         raw_cs = '/'.join((PROJECT, 'changeset', i.raw_id))
         if limit and limit == cnt:
             break
 
         full_uri = (BASE_URI % raw_cs)
         s = time.time()
         f = o.open(full_uri)
         size = len(f.read())
         e = time.time() - s
         total_time += e
         print '%s visited %s\%s size:%s req:%s ms' % (cnt, full_uri, i, size, e)
 
     print 'total_time', total_time
     print 'average on req', total_time / float(cnt)
 
 
 def test_files_walk(limit=100):
     print 'processing', jn(PROJECT_PATH, PROJECT)
     total_time = 0
 
     repo = vcs.get_repo(jn(PROJECT_PATH, PROJECT))
 
-    from rhodecode.lib.oset import OrderedSet
+    from rhodecode.lib.compat import OrderedSet
 
     paths_ = OrderedSet([''])
     try:
         tip = repo.get_changeset('tip')
         for topnode, dirs, files in tip.walk('/'):
 
             for dir in dirs:
                 paths_.add(dir.path)
                 for f in dir:
                     paths_.add(f.path)
 
             for f in files:
                 paths_.add(f.path)
 
     except vcs.exception.RepositoryError, e:
         pass
 
     cnt = 0
     for f in paths_:
         cnt += 1
         if limit and limit == cnt:
             break
 
         file_path = '/'.join((PROJECT, 'files', 'tip', f))
 
         full_uri = (BASE_URI % file_path)
         s = time.time()
         f = o.open(full_uri)
         size = len(f.read())
         e = time.time() - s
         total_time += e
         print '%s visited %s size:%s req:%s ms' % (cnt, full_uri, size, e)
 
     print 'total_time', total_time
     print 'average on req', total_time / float(cnt)
 
 
 
 test_changelog_walk(40)
 time.sleep(2)
 test_changeset_walk(limit=100)
 time.sleep(2)
 test_files_walk(100)

setup.py

@@ -1,125 +1,125 b''
 import sys
 from rhodecode import get_version
 from rhodecode import __platform__
 from rhodecode import __license__
 from rhodecode import PLATFORM_OTHERS
 
 py_version = sys.version_info
 
 if py_version < (2, 5):
     raise Exception('RhodeCode requires python 2.5 or later')
 
 requirements = [
         "Pylons==1.0.0",
         "WebHelpers>=1.2",
         "formencode==1.2.4",
         "SQLAlchemy>=0.7.2,<0.8",
         "Mako>=0.4.2",
         "pygments>=1.4",
         "mercurial>=1.9,<2.0",
         "whoosh<1.8",
         "celery>=2.2.5,<2.3",
         "babel",
         "python-dateutil>=1.5.0,<2.0.0",
         "dulwich>=0.8.0",
         "vcs>=0.2.1.dev",
-        "webob==1.0.8"    
+        "webob==1.0.8"
     ]
 
 dependency_links = [
-    "https://secure.rhodecode.org/vcs/archive/default.zip#egg=vcs-0.2.1.dev",
-    "https://bitbucket.org/marcinkuzminski/vcs/get/default.zip#egg=vcs-0.2.1.dev",
+    "https://secure.rhodecode.org/vcs/archive/default.zip#egg=vcs-0.2.2.dev",
+    "https://bitbucket.org/marcinkuzminski/vcs/get/default.zip#egg=vcs-0.2.2.dev",
 ]
 
 classifiers = ['Development Status :: 4 - Beta',
                'Environment :: Web Environment',
                'Framework :: Pylons',
                'Intended Audience :: Developers',
                'Operating System :: OS Independent',
                'Programming Language :: Python',
                'Programming Language :: Python :: 2.5',
                'Programming Language :: Python :: 2.6',
                'Programming Language :: Python :: 2.7', ]
 
 if py_version < (2, 6):
     requirements.append("simplejson")
     requirements.append("pysqlite")
 
 if __platform__ in PLATFORM_OTHERS:
     requirements.append("py-bcrypt")
 
 
 #additional files from project that goes somewhere in the filesystem
 #relative to sys.prefix
 data_files = []
 
 #additional files that goes into package itself
 package_data = {'rhodecode': ['i18n/*/LC_MESSAGES/*.mo', ], }
 
 description = ('Mercurial repository browser/management with '
                'build in push/pull server and full text search')
 keywords = ' '.join(['rhodecode', 'rhodiumcode', 'mercurial', 'git',
                       'repository management', 'hgweb replacement'
                       'hgwebdir', 'gitweb replacement', 'serving hgweb', ])
 #long description
 try:
     readme_file = 'README.rst'
     changelog_file = 'docs/changelog.rst'
     long_description = open(readme_file).read() + '\n\n' + \
         open(changelog_file).read()
 
 except IOError, err:
     sys.stderr.write("[WARNING] Cannot find file specified as "
         "long_description (%s)\n or changelog (%s) skipping that file" \
             % (readme_file, changelog_file))
     long_description = description
 
 
 try:
     from setuptools import setup, find_packages
 except ImportError:
     from ez_setup import use_setuptools
     use_setuptools()
     from setuptools import setup, find_packages
 #packages
 packages = find_packages(exclude=['ez_setup'])
 
 setup(
     name='RhodeCode',
     version=get_version(),
     description=description,
     long_description=long_description,
     keywords=keywords,
     license=__license__,
     author='Marcin Kuzminski',
     author_email='[e-mail]',
     dependency_links=dependency_links,
     url='http://rhodecode.org',
     install_requires=requirements,
     classifiers=classifiers,
     setup_requires=["PasteScript>=1.6.3"],
     data_files=data_files,
     packages=packages,
     include_package_data=True,
     test_suite='nose.collector',
     package_data=package_data,
     message_extractors={'rhodecode': [
             ('**.py', 'python', None),
             ('templates/**.mako', 'mako', {'input_encoding': 'utf-8'}),
             ('templates/**.html', 'mako', {'input_encoding': 'utf-8'}),
             ('public/**', 'ignore', None)]},
     zip_safe=False,
     paster_plugins=['PasteScript', 'Pylons'],
     entry_points="""
     [paste.app_factory]
     main = rhodecode.config.middleware:make_app
 
     [paste.app_install]
     main = pylons.util:PylonsInstaller
 
     [paste.global_paster_command]
     make-index = rhodecode.lib.indexers:MakeIndex
     upgrade-db = rhodecode.lib.dbmigrate:UpgradeDb
     celeryd=rhodecode.lib.celerypylons.commands:CeleryDaemonCommand
     """,
 )