- fixed issue with missing commits on some repos commands

- fixed issue with parsing commits for stats when there were only two commits

rhodecode/controllers/admin/repos.py

@@ -1,424 +1,430
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.admin.repos
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Admin controller for RhodeCode
 
     :created_on: Apr 7, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
 import traceback
 import formencode
 from formencode import htmlfill
 
 from paste.httpexceptions import HTTPInternalServerError
 from pylons import request, session, tmpl_context as c, url
 from pylons.controllers.util import redirect
 from pylons.i18n.translation import _
 from sqlalchemy.exc import IntegrityError
 
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator, \
     HasPermissionAnyDecorator
 from rhodecode.lib.base import BaseController, render
 from rhodecode.lib.utils import invalidate_cache, action_logger, repo_name_slug
 from rhodecode.lib.helpers import get_token
 from rhodecode.model.meta import Session
 from rhodecode.model.db import User, Repository, UserFollowing, RepoGroup
 from rhodecode.model.forms import RepoForm
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.repo import RepoModel
 
 log = logging.getLogger(__name__)
 
 
 class ReposController(BaseController):
     """
     REST Controller styled on the Atom Publishing Protocol"""
     # To properly map this controller, ensure your config/routing.py
     # file has a resource setup:
     #     map.resource('repo', 'repos')
 
     @LoginRequired()
     @HasPermissionAnyDecorator('hg.admin', 'hg.create.repository')
     def __before__(self):
         c.admin_user = session.get('admin_user')
         c.admin_username = session.get('admin_username')
         super(ReposController, self).__before__()
 
     def __load_defaults(self):
         c.repo_groups = RepoGroup.groups_choices()
         c.repo_groups_choices = map(lambda k: unicode(k[0]), c.repo_groups)
 
         repo_model = RepoModel()
         c.users_array = repo_model.get_users_js()
         c.users_groups_array = repo_model.get_users_groups_js()
 
     def __load_data(self, repo_name=None):
         """
         Load defaults settings for edit, and update
 
         :param repo_name:
         """
         self.__load_defaults()
 
         c.repo_info = db_repo = Repository.get_by_repo_name(repo_name)
         repo = db_repo.scm_instance
 
         if c.repo_info is None:
             h.flash(_('%s repository is not mapped to db perhaps'
                       ' it was created or renamed from the filesystem'
                       ' please run the application again'
                       ' in order to rescan repositories') % repo_name,
                       category='error')
 
             return redirect(url('repos'))
 
         c.default_user_id = User.get_by_username('default').user_id
         c.in_public_journal = UserFollowing.query()\
             .filter(UserFollowing.user_id == c.default_user_id)\
             .filter(UserFollowing.follows_repository == c.repo_info).scalar()
 
         if c.repo_info.stats:
-            last_rev = c.repo_info.stats.stat_on_revision
+            # this is on what revision we ended up so we add +1 for count
+            last_rev = c.repo_info.stats.stat_on_revision + 1
         else:
             last_rev = 0
         c.stats_revision = last_rev
 
-        c.repo_last_rev = repo.count() - 1 if repo.revisions else 0
+        c.repo_last_rev = repo.count() if repo.revisions else 0
 
         if last_rev == 0 or c.repo_last_rev == 0:
             c.stats_percentage = 0
         else:
             c.stats_percentage = '%.2f' % ((float((last_rev)) /
                                             c.repo_last_rev) * 100)
 
         defaults = RepoModel()._get_defaults(repo_name)
         
         c.repos_list = [('', _('--REMOVE FORK--'))]
         c.repos_list += [(x.repo_id, x.repo_name) for x in
                    Repository.query().order_by(Repository.repo_name).all()]
         return defaults
 
     @HasPermissionAllDecorator('hg.admin')
     def index(self, format='html'):
         """GET /repos: All items in the collection"""
         # url('repos')
 
         c.repos_list = ScmModel().get_repos(Repository.query()
                                             .order_by(Repository.repo_name)
                                             .all(), sort_key='name_sort')
         return render('admin/repos/repos.html')
 
     @HasPermissionAnyDecorator('hg.admin', 'hg.create.repository')
     def create(self):
         """
         POST /repos: Create a new item"""
         # url('repos')
 
         self.__load_defaults()
         form_result = {}
         try:
             form_result = RepoForm(repo_groups=c.repo_groups_choices)()\
                             .to_python(dict(request.POST))
             RepoModel().create(form_result, self.rhodecode_user)
             if form_result['clone_uri']:
                 h.flash(_('created repository %s from %s') \
                     % (form_result['repo_name'], form_result['clone_uri']),
                     category='success')
             else:
                 h.flash(_('created repository %s') % form_result['repo_name'],
                     category='success')
 
             if request.POST.get('user_created'):
                 # created by regular non admin user
                 action_logger(self.rhodecode_user, 'user_created_repo',
                               form_result['repo_name_full'], '', self.sa)
             else:
                 action_logger(self.rhodecode_user, 'admin_created_repo',
                               form_result['repo_name_full'], '', self.sa)
             Session.commit()
         except formencode.Invalid, errors:
 
             c.new_repo = errors.value['repo_name']
 
             if request.POST.get('user_created'):
                 r = render('admin/repos/repo_add_create_repository.html')
             else:
                 r = render('admin/repos/repo_add.html')
 
             return htmlfill.render(
                 r,
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8")
 
         except Exception:
             log.error(traceback.format_exc())
             msg = _('error occurred during creation of repository %s') \
                     % form_result.get('repo_name')
             h.flash(msg, category='error')
         if request.POST.get('user_created'):
             return redirect(url('home'))
         return redirect(url('repos'))
 
     @HasPermissionAllDecorator('hg.admin')
     def new(self, format='html'):
         """GET /repos/new: Form to create a new item"""
         new_repo = request.GET.get('repo', '')
         c.new_repo = repo_name_slug(new_repo)
         self.__load_defaults()
         return render('admin/repos/repo_add.html')
 
     @HasPermissionAllDecorator('hg.admin')
     def update(self, repo_name):
         """
         PUT /repos/repo_name: Update an existing item"""
         # Forms posted to this method should contain a hidden field:
         #    <input type="hidden" name="_method" value="PUT" />
         # Or using helpers:
         #    h.form(url('repo', repo_name=ID),
         #           method='put')
         # url('repo', repo_name=ID)
         self.__load_defaults()
         repo_model = RepoModel()
         changed_name = repo_name
         _form = RepoForm(edit=True, old_data={'repo_name': repo_name},
                          repo_groups=c.repo_groups_choices)()
         try:
             form_result = _form.to_python(dict(request.POST))
             repo = repo_model.update(repo_name, form_result)
             invalidate_cache('get_repo_cached_%s' % repo_name)
             h.flash(_('Repository %s updated successfully' % repo_name),
                     category='success')
             changed_name = repo.repo_name
             action_logger(self.rhodecode_user, 'admin_updated_repo',
                               changed_name, '', self.sa)
             Session.commit()
         except formencode.Invalid, errors:
             defaults = self.__load_data(repo_name)
             defaults.update(errors.value)
             return htmlfill.render(
                 render('admin/repos/repo_edit.html'),
                 defaults=defaults,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8")
 
         except Exception:
             log.error(traceback.format_exc())
             h.flash(_('error occurred during update of repository %s') \
                     % repo_name, category='error')
         return redirect(url('edit_repo', repo_name=changed_name))
 
     @HasPermissionAllDecorator('hg.admin')
     def delete(self, repo_name):
         """
         DELETE /repos/repo_name: Delete an existing item"""
         # Forms posted to this method should contain a hidden field:
         #    <input type="hidden" name="_method" value="DELETE" />
         # Or using helpers:
         #    h.form(url('repo', repo_name=ID),
         #           method='delete')
         # url('repo', repo_name=ID)
 
         repo_model = RepoModel()
         repo = repo_model.get_by_repo_name(repo_name)
         if not repo:
             h.flash(_('%s repository is not mapped to db perhaps'
                       ' it was moved or renamed  from the filesystem'
                       ' please run the application again'
                       ' in order to rescan repositories') % repo_name,
                       category='error')
 
             return redirect(url('repos'))
         try:
             action_logger(self.rhodecode_user, 'admin_deleted_repo',
                               repo_name, '', self.sa)
             repo_model.delete(repo)
             invalidate_cache('get_repo_cached_%s' % repo_name)
             h.flash(_('deleted repository %s') % repo_name, category='success')
             Session.commit()
         except IntegrityError, e:
             if e.message.find('repositories_fork_id_fkey') != -1:
                 log.error(traceback.format_exc())
                 h.flash(_('Cannot delete %s it still contains attached '
                           'forks') % repo_name,
                         category='warning')
             else:
                 log.error(traceback.format_exc())
                 h.flash(_('An error occurred during '
                           'deletion of %s') % repo_name,
                         category='error')
 
         except Exception, e:
             log.error(traceback.format_exc())
             h.flash(_('An error occurred during deletion of %s') % repo_name,
                     category='error')
 
         return redirect(url('repos'))
 
     @HasPermissionAllDecorator('hg.admin')
     def delete_perm_user(self, repo_name):
         """
         DELETE an existing repository permission user
 
         :param repo_name:
         """
 
         try:
             repo_model = RepoModel()
             repo_model.delete_perm_user(request.POST, repo_name)
+            Session.commit()
         except Exception, e:
             h.flash(_('An error occurred during deletion of repository user'),
                     category='error')
             raise HTTPInternalServerError()
 
     @HasPermissionAllDecorator('hg.admin')
     def delete_perm_users_group(self, repo_name):
         """
         DELETE an existing repository permission users group
 
         :param repo_name:
         """
         try:
             repo_model = RepoModel()
             repo_model.delete_perm_users_group(request.POST, repo_name)
+            Session.commit()
         except Exception, e:
             h.flash(_('An error occurred during deletion of repository'
                       ' users groups'),
                     category='error')
             raise HTTPInternalServerError()
 
     @HasPermissionAllDecorator('hg.admin')
     def repo_stats(self, repo_name):
         """
         DELETE an existing repository statistics
 
         :param repo_name:
         """
 
         try:
             repo_model = RepoModel()
             repo_model.delete_stats(repo_name)
+            Session.commit()
         except Exception, e:
             h.flash(_('An error occurred during deletion of repository stats'),
                     category='error')
         return redirect(url('edit_repo', repo_name=repo_name))
 
     @HasPermissionAllDecorator('hg.admin')
     def repo_cache(self, repo_name):
         """
         INVALIDATE existing repository cache
 
         :param repo_name:
         """
 
         try:
             ScmModel().mark_for_invalidation(repo_name)
+            Session.commit()
         except Exception, e:
             h.flash(_('An error occurred during cache invalidation'),
                     category='error')
         return redirect(url('edit_repo', repo_name=repo_name))
 
     @HasPermissionAllDecorator('hg.admin')
     def repo_public_journal(self, repo_name):
         """
         Set's this repository to be visible in public journal,
         in other words assing default user to follow this repo
 
         :param repo_name:
         """
 
         cur_token = request.POST.get('auth_token')
         token = get_token()
         if cur_token == token:
             try:
                 repo_id = Repository.get_by_repo_name(repo_name).repo_id
                 user_id = User.get_by_username('default').user_id
                 self.scm_model.toggle_following_repo(repo_id, user_id)
                 h.flash(_('Updated repository visibility in public journal'),
                         category='success')
+                Session.commit()
             except:
                 h.flash(_('An error occurred during setting this'
                           ' repository in public journal'),
                         category='error')
 
         else:
             h.flash(_('Token mismatch'), category='error')
         return redirect(url('edit_repo', repo_name=repo_name))
 
     @HasPermissionAllDecorator('hg.admin')
     def repo_pull(self, repo_name):
         """
         Runs task to update given repository with remote changes,
         ie. make pull on remote location
 
         :param repo_name:
         """
         try:
             ScmModel().pull_changes(repo_name, self.rhodecode_user.username)
             h.flash(_('Pulled from remote location'), category='success')
         except Exception, e:
             h.flash(_('An error occurred during pull from remote location'),
                     category='error')
 
         return redirect(url('edit_repo', repo_name=repo_name))
 
     @HasPermissionAllDecorator('hg.admin')
     def repo_as_fork(self, repo_name):
         """
         Mark given repository as a fork of another
         
         :param repo_name:
         """
         try:
             fork_id = request.POST.get('id_fork_of')
             repo = ScmModel().mark_as_fork(repo_name, fork_id,
                                     self.rhodecode_user.username)
             fork = repo.fork.repo_name if repo.fork else _('Nothing')
             Session.commit()
             h.flash(_('Marked repo %s as fork of %s' % (repo_name,fork)), 
                     category='success')
         except Exception, e:
             raise
             h.flash(_('An error occurred during this operation'),
                     category='error')
 
         return redirect(url('edit_repo', repo_name=repo_name))
 
     @HasPermissionAllDecorator('hg.admin')
     def show(self, repo_name, format='html'):
         """GET /repos/repo_name: Show a specific item"""
         # url('repo', repo_name=ID)
 
     @HasPermissionAllDecorator('hg.admin')
     def edit(self, repo_name, format='html'):
         """GET /repos/repo_name/edit: Form to edit an existing item"""
         # url('edit_repo', repo_name=ID)
         defaults = self.__load_data(repo_name)
 
         return htmlfill.render(
             render('admin/repos/repo_edit.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )

rhodecode/controllers/forks.py

@@ -1,174 +1,174
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.forks
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     forks controller for rhodecode
 
     :created_on: Apr 23, 2011
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import logging
 import formencode
 import traceback
 from formencode import htmlfill
 
 from pylons import tmpl_context as c, request, url
 from pylons.controllers.util import redirect
 from pylons.i18n.translation import _
 
 import rhodecode.lib.helpers as h
 
 from rhodecode.lib.helpers import Page
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator, \
     NotAnonymous
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.model.db import Repository, RepoGroup, UserFollowing, User
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.forms import RepoForkForm
 
 log = logging.getLogger(__name__)
 
 
 class ForksController(BaseRepoController):
 
     @LoginRequired()
     def __before__(self):
         super(ForksController, self).__before__()
 
     def __load_defaults(self):
         c.repo_groups = RepoGroup.groups_choices()
         c.repo_groups_choices = map(lambda k: unicode(k[0]), c.repo_groups)
 
     def __load_data(self, repo_name=None):
         """
         Load defaults settings for edit, and update
 
         :param repo_name:
         """
         self.__load_defaults()
 
         c.repo_info = db_repo = Repository.get_by_repo_name(repo_name)
         repo = db_repo.scm_instance
 
         if c.repo_info is None:
             h.flash(_('%s repository is not mapped to db perhaps'
                       ' it was created or renamed from the filesystem'
                       ' please run the application again'
                       ' in order to rescan repositories') % repo_name,
                       category='error')
 
             return redirect(url('repos'))
 
         c.default_user_id = User.get_by_username('default').user_id
         c.in_public_journal = UserFollowing.query()\
             .filter(UserFollowing.user_id == c.default_user_id)\
             .filter(UserFollowing.follows_repository == c.repo_info).scalar()
 
         if c.repo_info.stats:
-            last_rev = c.repo_info.stats.stat_on_revision
+            last_rev = c.repo_info.stats.stat_on_revision+1
         else:
             last_rev = 0
         c.stats_revision = last_rev
 
-        c.repo_last_rev = repo.count() - 1 if repo.revisions else 0
+        c.repo_last_rev = repo.count() if repo.revisions else 0
 
         if last_rev == 0 or c.repo_last_rev == 0:
             c.stats_percentage = 0
         else:
             c.stats_percentage = '%.2f' % ((float((last_rev)) /
                                             c.repo_last_rev) * 100)
 
         defaults = RepoModel()._get_defaults(repo_name)
         # add prefix to fork
         defaults['repo_name'] = 'fork-' + defaults['repo_name']
         return defaults
 
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def forks(self, repo_name):
         p = int(request.params.get('page', 1))
         repo_id = c.rhodecode_db_repo.repo_id
         d = Repository.get_repo_forks(repo_id)
         c.forks_pager = Page(d, page=p, items_per_page=20)
 
         c.forks_data = render('/forks/forks_data.html')
 
         if request.environ.get('HTTP_X_PARTIAL_XHR'):
             return c.forks_data
 
         return render('/forks/forks.html')
 
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def fork(self, repo_name):
         c.repo_info = Repository.get_by_repo_name(repo_name)
         if not c.repo_info:
             h.flash(_('%s repository is not mapped to db perhaps'
                       ' it was created or renamed from the file system'
                       ' please run the application again'
                       ' in order to rescan repositories') % repo_name,
                       category='error')
 
             return redirect(url('home'))
 
         defaults = self.__load_data(repo_name)
 
         return htmlfill.render(
             render('forks/fork.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
 
 
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def fork_create(self, repo_name):
         self.__load_defaults()
         c.repo_info = Repository.get_by_repo_name(repo_name)
         _form = RepoForkForm(old_data={'repo_type': c.repo_info.repo_type},
                              repo_groups=c.repo_groups_choices,)()
         form_result = {}
         try:
             form_result = _form.to_python(dict(request.POST))
             # add org_path of repo so we can do a clone from it later
             form_result['org_path'] = c.repo_info.repo_name
 
             # create fork is done sometimes async on celery, db transaction
             # management is handled there.
             RepoModel().create_fork(form_result, self.rhodecode_user)
             h.flash(_('forked %s repository as %s') \
                       % (repo_name, form_result['repo_name']),
                     category='success')
         except formencode.Invalid, errors:
             c.new_repo = errors.value['repo_name']
 
             return htmlfill.render(
                 render('forks/fork.html'),
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8")
         except Exception:
             log.error(traceback.format_exc())
             h.flash(_('An error occurred during repository forking %s') %
                     repo_name, category='error')
 
         return redirect(url('home'))

rhodecode/controllers/summary.py

@@ -1,228 +1,229
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.summary
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Summary controller for Rhodecode
 
     :created_on: Apr 18, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import traceback
 import calendar
 import logging
 from time import mktime
 from datetime import timedelta, date
 from itertools import product
 from urlparse import urlparse
 
 from vcs.exceptions import ChangesetError, EmptyRepositoryError, \
     NodeDoesNotExistError
 
 from pylons import tmpl_context as c, request, url, config
 from pylons.i18n.translation import _
 
 from beaker.cache import cache_region, region_invalidate
 
 from rhodecode.model.db import Statistics, CacheInvalidation
 from rhodecode.lib import ALL_READMES, ALL_EXTS
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.utils import EmptyChangeset
 from rhodecode.lib.markup_renderer import MarkupRenderer
 from rhodecode.lib.celerylib import run_task
 from rhodecode.lib.celerylib.tasks import get_commits_stats, \
     LANGUAGES_EXTENSIONS_MAP
 from rhodecode.lib.helpers import RepoPage
 from rhodecode.lib.compat import json, OrderedDict
 
 log = logging.getLogger(__name__)
 
 README_FILES = [''.join([x[0][0], x[1][0]]) for x in
                     sorted(list(product(ALL_READMES, ALL_EXTS)),
                            key=lambda y:y[0][1] + y[1][1])]
 
+
 class SummaryController(BaseRepoController):
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def __before__(self):
         super(SummaryController, self).__before__()
 
     def index(self, repo_name):
         c.dbrepo = dbrepo = c.rhodecode_db_repo
         c.following = self.scm_model.is_following_repo(repo_name,
                                                 self.rhodecode_user.user_id)
 
         def url_generator(**kw):
             return url('shortlog_home', repo_name=repo_name, size=10, **kw)
 
         c.repo_changesets = RepoPage(c.rhodecode_repo, page=1,
                                      items_per_page=10, url=url_generator)
 
         if self.rhodecode_user.username == 'default':
             # for default(anonymous) user we don't need to pass credentials
             username = ''
             password = ''
         else:
             username = str(self.rhodecode_user.username)
             password = '@'
 
         parsed_url = urlparse(url.current(qualified=True))
 
         default_clone_uri = '{scheme}://{user}{pass}{netloc}{path}'
 
         uri_tmpl = config.get('clone_uri', default_clone_uri)
         uri_tmpl = uri_tmpl.replace('{', '%(').replace('}', ')s')
 
         uri = uri_tmpl % {'user': username,
                            'pass': password,
                            'scheme': parsed_url.scheme,
-                           'netloc':parsed_url.netloc,
+                           'netloc': parsed_url.netloc,
                            'path':parsed_url.path}
 
         c.clone_repo_url = uri
         c.repo_tags = OrderedDict()
         for name, hash in c.rhodecode_repo.tags.items()[:10]:
             try:
                 c.repo_tags[name] = c.rhodecode_repo.get_changeset(hash)
             except ChangesetError:
                 c.repo_tags[name] = EmptyChangeset(hash)
 
         c.repo_branches = OrderedDict()
         for name, hash in c.rhodecode_repo.branches.items()[:10]:
             try:
                 c.repo_branches[name] = c.rhodecode_repo.get_changeset(hash)
             except ChangesetError:
                 c.repo_branches[name] = EmptyChangeset(hash)
 
         td = date.today() + timedelta(days=1)
         td_1m = td - timedelta(days=calendar.mdays[td.month])
         td_1y = td - timedelta(days=365)
 
         ts_min_m = mktime(td_1m.timetuple())
         ts_min_y = mktime(td_1y.timetuple())
         ts_max_y = mktime(td.timetuple())
 
         if dbrepo.enable_statistics:
             c.show_stats = True
             c.no_data_msg = _('No data loaded yet')
             run_task(get_commits_stats, c.dbrepo.repo_name, ts_min_y, ts_max_y)
         else:
             c.show_stats = False
             c.no_data_msg = _('Statistics are disabled for this repository')
         c.ts_min = ts_min_m
         c.ts_max = ts_max_y
 
         stats = self.sa.query(Statistics)\
             .filter(Statistics.repository == dbrepo)\
             .scalar()
 
         c.stats_percentage = 0
 
         if stats and stats.languages:
             c.no_data = False is dbrepo.enable_statistics
             lang_stats_d = json.loads(stats.languages)
             c.commit_data = stats.commit_activity
             c.overview_data = stats.commit_activity_combined
 
             lang_stats = ((x, {"count": y,
                                "desc": LANGUAGES_EXTENSIONS_MAP.get(x)})
                           for x, y in lang_stats_d.items())
 
             c.trending_languages = json.dumps(OrderedDict(
                                        sorted(lang_stats, reverse=True,
                                             key=lambda k: k[1])[:10]
                                         )
                                     )
-            last_rev = stats.stat_on_revision
-            c.repo_last_rev = c.rhodecode_repo.count() - 1 \
+            last_rev = stats.stat_on_revision + 1
+            c.repo_last_rev = c.rhodecode_repo.count()\
                 if c.rhodecode_repo.revisions else 0
             if last_rev == 0 or c.repo_last_rev == 0:
                 pass
             else:
                 c.stats_percentage = '%.2f' % ((float((last_rev)) /
                                                 c.repo_last_rev) * 100)
         else:
             c.commit_data = json.dumps({})
             c.overview_data = json.dumps([[ts_min_y, 0], [ts_max_y, 10]])
             c.trending_languages = json.dumps({})
             c.no_data = True
 
         c.enable_downloads = dbrepo.enable_downloads
         if c.enable_downloads:
             c.download_options = self._get_download_links(c.rhodecode_repo)
 
         c.readme_data, c.readme_file = self.__get_readme_data(c.rhodecode_repo)
         return render('summary/summary.html')
 
     def __get_readme_data(self, repo):
 
         @cache_region('long_term')
         def _get_readme_from_cache(key):
             readme_data = None
             readme_file = None
             log.debug('Fetching readme file')
             try:
                 cs = repo.get_changeset('tip')
                 renderer = MarkupRenderer()
                 for f in README_FILES:
                     try:
                         readme = cs.get_node(f)
                         readme_file = f
                         readme_data = renderer.render(readme.content, f)
                         log.debug('Found readme %s' % readme_file)
                         break
                     except NodeDoesNotExistError:
                         continue
             except ChangesetError:
                 pass
             except EmptyRepositoryError:
                 pass
             except Exception:
                 log.error(traceback.format_exc())
 
             return readme_data, readme_file
 
         key = repo.name + '_README'
         inv = CacheInvalidation.invalidate(key)
         if inv is not None:
             region_invalidate(_get_readme_from_cache, None, key)
             CacheInvalidation.set_valid(inv.cache_key)
         return _get_readme_from_cache(key)
 
     def _get_download_links(self, repo):
 
         download_l = []
 
         branches_group = ([], _("Branches"))
         tags_group = ([], _("Tags"))
 
         for name, chs in c.rhodecode_repo.branches.items():
             #chs = chs.split(':')[-1]
             branches_group[0].append((chs, name),)
         download_l.append(branches_group)
 
         for name, chs in c.rhodecode_repo.tags.items():
             #chs = chs.split(':')[-1]
             tags_group[0].append((chs, name),)
         download_l.append(tags_group)
 
         return download_l
 

rhodecode/lib/celerylib/tasks.py

@@ -1,414 +1,418
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.celerylib.tasks
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     RhodeCode task modules, containing all task that suppose to be run
     by celery daemon
 
     :created_on: Oct 6, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 from celery.decorators import task
 
 import os
 import traceback
 import logging
 from os.path import join as jn
 
 from time import mktime
 from operator import itemgetter
 from string import lower
 
 from pylons import config, url
 from pylons.i18n.translation import _
 
 from vcs import get_backend
 
 from rhodecode import CELERY_ON
 from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP, safe_str
 from rhodecode.lib.celerylib import run_task, locked_task, str2bool, \
     __get_lockkey, LockHeld, DaemonLock
 from rhodecode.lib.helpers import person
 from rhodecode.lib.rcmail.smtp_mailer import SmtpMailer
 from rhodecode.lib.utils import add_cache, action_logger
 from rhodecode.lib.compat import json, OrderedDict
 
 from rhodecode.model import init_model
 from rhodecode.model import meta
 from rhodecode.model.db import Statistics, Repository, User
 
 from sqlalchemy import engine_from_config
 
 add_cache(config)
 
 __all__ = ['whoosh_index', 'get_commits_stats',
            'reset_user_password', 'send_email']
 
 
 def get_session():
     if CELERY_ON:
         engine = engine_from_config(config, 'sqlalchemy.db1.')
         init_model(engine)
     sa = meta.Session
     return sa
 
 def get_logger(cls):
     if CELERY_ON:
         try:
             log = cls.get_logger()
         except:
             log = logging.getLogger(__name__)
     else:
         log = logging.getLogger(__name__)
 
     return log
 
+
 @task(ignore_result=True)
 @locked_task
 def whoosh_index(repo_location, full_index):
     from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
 
-    #log = whoosh_index.get_logger()
+    # log = whoosh_index.get_logger(whoosh_index)
 
     index_location = config['index_dir']
     WhooshIndexingDaemon(index_location=index_location,
                          repo_location=repo_location, sa=get_session())\
                          .run(full_index=full_index)
 
 
 @task(ignore_result=True)
 def get_commits_stats(repo_name, ts_min_y, ts_max_y):
     log = get_logger(get_commits_stats)
 
     lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y,
                             ts_max_y)
     lockkey_path = config['here']
 
     log.info('running task with lockkey %s', lockkey)
+
     try:
         sa = get_session()
         lock = l = DaemonLock(file_=jn(lockkey_path, lockkey))
 
         # for js data compatibilty cleans the key for person from '
         akc = lambda k: person(k).replace('"', "")
 
         co_day_auth_aggr = {}
         commits_by_day_aggregate = {}
         repo = Repository.get_by_repo_name(repo_name)
         if repo is None:
             return True
         
         repo = repo.scm_instance
-        repo_size = len(repo.revisions)
-        #return if repo have no revisions
+        repo_size = repo.count()
+        # return if repo have no revisions
         if repo_size < 1:
             lock.release()
             return True
 
         skip_date_limit = True
         parse_limit = int(config['app_conf'].get('commit_parse_limit'))
-        last_rev = 0
+        last_rev = None
         last_cs = None
         timegetter = itemgetter('time')
 
         dbrepo = sa.query(Repository)\
             .filter(Repository.repo_name == repo_name).scalar()
         cur_stats = sa.query(Statistics)\
             .filter(Statistics.repository == dbrepo).scalar()
 
         if cur_stats is not None:
             last_rev = cur_stats.stat_on_revision
 
         if last_rev == repo.get_changeset().revision and repo_size > 1:
             # pass silently without any work if we're not on first revision or
             # current state of parsing revision(from db marker) is the
             # last revision
             lock.release()
             return True
 
         if cur_stats:
             commits_by_day_aggregate = OrderedDict(json.loads(
                                         cur_stats.commit_activity_combined))
             co_day_auth_aggr = json.loads(cur_stats.commit_activity)
 
         log.debug('starting parsing %s', parse_limit)
         lmktime = mktime
 
-        last_rev = last_rev + 1 if last_rev > 0 else last_rev
-
+        last_rev = last_rev + 1 if last_rev >= 0 else 0
+        log.debug('Getting revisions from %s to %s' % (
+             last_rev, last_rev + parse_limit)
+        )
         for cs in repo[last_rev:last_rev + parse_limit]:
             last_cs = cs  # remember last parsed changeset
             k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1],
                           cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0])
 
             if akc(cs.author) in co_day_auth_aggr:
                 try:
                     l = [timegetter(x) for x in
                          co_day_auth_aggr[akc(cs.author)]['data']]
                     time_pos = l.index(k)
                 except ValueError:
                     time_pos = False
 
                 if time_pos >= 0 and time_pos is not False:
 
                     datadict = \
                         co_day_auth_aggr[akc(cs.author)]['data'][time_pos]
 
                     datadict["commits"] += 1
                     datadict["added"] += len(cs.added)
                     datadict["changed"] += len(cs.changed)
                     datadict["removed"] += len(cs.removed)
 
                 else:
                     if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
 
                         datadict = {"time": k,
                                     "commits": 1,
                                     "added": len(cs.added),
                                     "changed": len(cs.changed),
                                     "removed": len(cs.removed),
                                    }
                         co_day_auth_aggr[akc(cs.author)]['data']\
                             .append(datadict)
 
             else:
                 if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
                     co_day_auth_aggr[akc(cs.author)] = {
                                         "label": akc(cs.author),
                                         "data": [{"time":k,
                                                  "commits":1,
                                                  "added":len(cs.added),
                                                  "changed":len(cs.changed),
                                                  "removed":len(cs.removed),
                                                  }],
                                         "schema": ["commits"],
                                         }
 
             #gather all data by day
             if k in commits_by_day_aggregate:
                 commits_by_day_aggregate[k] += 1
             else:
                 commits_by_day_aggregate[k] = 1
 
         overview_data = sorted(commits_by_day_aggregate.items(),
                                key=itemgetter(0))
 
         if not co_day_auth_aggr:
             co_day_auth_aggr[akc(repo.contact)] = {
                 "label": akc(repo.contact),
                 "data": [0, 1],
                 "schema": ["commits"],
             }
 
         stats = cur_stats if cur_stats else Statistics()
         stats.commit_activity = json.dumps(co_day_auth_aggr)
         stats.commit_activity_combined = json.dumps(overview_data)
 
         log.debug('last revison %s', last_rev)
         leftovers = len(repo.revisions[last_rev:])
         log.debug('revisions to parse %s', leftovers)
 
         if last_rev == 0 or leftovers < parse_limit:
             log.debug('getting code trending stats')
             stats.languages = json.dumps(__get_codes_stats(repo_name))
 
         try:
             stats.repository = dbrepo
             stats.stat_on_revision = last_cs.revision if last_cs else 0
             sa.add(stats)
             sa.commit()
         except:
             log.error(traceback.format_exc())
             sa.rollback()
             lock.release()
             return False
 
         #final release
         lock.release()
 
         #execute another task if celery is enabled
         if len(repo.revisions) > 1 and CELERY_ON:
             run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
         return True
     except LockHeld:
         log.info('LockHeld')
         return 'Task with key %s already running' % lockkey
 
 @task(ignore_result=True)
 def send_password_link(user_email):
     from rhodecode.model.notification import EmailNotificationModel
 
     log = get_logger(send_password_link)
 
     try:
         sa = get_session()
         user = User.get_by_email(user_email)
         if user:
             log.debug('password reset user found %s' % user)
             link = url('reset_password_confirmation', key=user.api_key,
                        qualified=True)
             reg_type = EmailNotificationModel.TYPE_PASSWORD_RESET
             body = EmailNotificationModel().get_email_tmpl(reg_type,
                                                 **{'user':user.short_contact,
                                                    'reset_url':link})
             log.debug('sending email')
             run_task(send_email, user_email,
                      _("password reset link"), body)
             log.info('send new password mail to %s', user_email)
         else:
             log.debug("password reset email %s not found" % user_email)
     except:
         log.error(traceback.format_exc())
         return False
 
     return True
 
 @task(ignore_result=True)
 def reset_user_password(user_email):
     from rhodecode.lib import auth
 
     log = get_logger(reset_user_password)
 
     try:
         try:
             sa = get_session()
             user = User.get_by_email(user_email)
             new_passwd = auth.PasswordGenerator().gen_password(8,
                              auth.PasswordGenerator.ALPHABETS_BIG_SMALL)
             if user:
                 user.password = auth.get_crypt_password(new_passwd)
                 user.api_key = auth.generate_api_key(user.username)
                 sa.add(user)
                 sa.commit()
                 log.info('change password for %s', user_email)
             if new_passwd is None:
                 raise Exception('unable to generate new password')
         except:
             log.error(traceback.format_exc())
             sa.rollback()
 
         run_task(send_email, user_email,
                  'Your new password',
                  'Your new RhodeCode password:%s' % (new_passwd))
         log.info('send new password mail to %s', user_email)
 
     except:
         log.error('Failed to update user password')
         log.error(traceback.format_exc())
 
     return True
 
 
 @task(ignore_result=True)
 def send_email(recipients, subject, body, html_body=''):
     """
     Sends an email with defined parameters from the .ini files.
 
     :param recipients: list of recipients, it this is empty the defined email
         address from field 'email_to' is used instead
     :param subject: subject of the mail
     :param body: body of the mail
     :param html_body: html version of body
     """
     log = get_logger(send_email)
     sa = get_session()
     email_config = config
     subject = "%s %s" % (email_config.get('email_prefix'), subject)
     if not recipients:
         # if recipients are not defined we send to email_config + all admins
         admins = [u.email for u in User.query()
                   .filter(User.admin == True).all()]
         recipients = [email_config.get('email_to')] + admins
 
     mail_from = email_config.get('app_email_from', 'RhodeCode')
     user = email_config.get('smtp_username')
     passwd = email_config.get('smtp_password')
     mail_server = email_config.get('smtp_server')
     mail_port = email_config.get('smtp_port')
     tls = str2bool(email_config.get('smtp_use_tls'))
     ssl = str2bool(email_config.get('smtp_use_ssl'))
     debug = str2bool(config.get('debug'))
     smtp_auth = email_config.get('smtp_auth')
 
     try:
         m = SmtpMailer(mail_from, user, passwd, mail_server, smtp_auth,
                        mail_port, ssl, tls, debug=debug)
         m.send(recipients, subject, body, html_body)
     except:
         log.error('Mail sending failed')
         log.error(traceback.format_exc())
         return False
     return True
 
 
 @task(ignore_result=True)
 def create_repo_fork(form_data, cur_user):
     """
     Creates a fork of repository using interval VCS methods
     
     :param form_data:
     :param cur_user:
     """
     from rhodecode.model.repo import RepoModel
 
     log = get_logger(create_repo_fork)
 
     Session = get_session()
     base_path = Repository.base_path()
 
     RepoModel(Session).create(form_data, cur_user, just_db=True, fork=True)
 
     alias = form_data['repo_type']
     org_repo_name = form_data['org_path']
     fork_name = form_data['repo_name_full']
     update_after_clone = form_data['update_after_clone']
     source_repo_path = os.path.join(base_path, org_repo_name)
     destination_fork_path = os.path.join(base_path, fork_name)
 
     log.info('creating fork of %s as %s', source_repo_path,
              destination_fork_path)
     backend = get_backend(alias)
     backend(safe_str(destination_fork_path), create=True,
             src_url=safe_str(source_repo_path),
             update_after_clone=update_after_clone)
     action_logger(cur_user, 'user_forked_repo:%s' % fork_name,
                    org_repo_name, '', Session)
     
     action_logger(cur_user, 'user_created_fork:%s' % fork_name,
                    fork_name, '', Session)    
     # finally commit at latest possible stage
     Session.commit()
 
 def __get_codes_stats(repo_name):
     repo = Repository.get_by_repo_name(repo_name).scm_instance
 
     tip = repo.get_changeset()
     code_stats = {}
 
     def aggregate(cs):
         for f in cs[2]:
             ext = lower(f.extension)
             if ext in LANGUAGES_EXTENSIONS_MAP.keys() and not f.is_binary:
                 if ext in code_stats:
                     code_stats[ext] += 1
                 else:
                     code_stats[ext] = 1
 
     map(aggregate, tip.walk('/'))
 
     return code_stats or {}