notifications: insert 'References' mail headers to help MUA threading

gmail is however fubar.

kallithea/lib/celerylib/tasks.py

@@ -1,506 +1,506 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.celerylib.tasks
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Kallithea task modules, containing all task that suppose to be run
 by celery daemon
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Oct 6, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 from celery.decorators import task
 
 import os
 import traceback
 import logging
 from os.path import join as jn
 
 from time import mktime
 from operator import itemgetter
 from string import lower
 
 from pylons import config, url
 from pylons.i18n.translation import _
 
 from kallithea.lib.vcs import get_backend
 
 from kallithea import CELERY_ON, CELERY_EAGER
 from kallithea.lib.utils2 import safe_str
 from kallithea.lib.celerylib import run_task, locked_task, dbsession, \
     str2bool, __get_lockkey, LockHeld, DaemonLock, get_session
 from kallithea.lib.helpers import person
 from kallithea.lib.rcmail.smtp_mailer import SmtpMailer
 from kallithea.lib.utils import add_cache, action_logger
 from kallithea.lib.compat import json, OrderedDict
 from kallithea.lib.hooks import log_create_repository
 
 from kallithea.model.db import Statistics, Repository, User
 from kallithea.model.scm import ScmModel
 
 
 add_cache(config)  # pragma: no cover
 
 __all__ = ['whoosh_index', 'get_commits_stats',
            'reset_user_password', 'send_email']
 
 
 def get_logger(cls):
     if CELERY_ON:
         try:
             log = cls.get_logger()
         except Exception:
             log = logging.getLogger(__name__)
     else:
         log = logging.getLogger(__name__)
 
     return log
 
 
 @task(ignore_result=True)
 @locked_task
 @dbsession
 def whoosh_index(repo_location, full_index):
     from kallithea.lib.indexers.daemon import WhooshIndexingDaemon
     log = get_logger(whoosh_index)
     DBS = get_session()
 
     index_location = config['index_dir']
     WhooshIndexingDaemon(index_location=index_location,
                          repo_location=repo_location, sa=DBS)\
                          .run(full_index=full_index)
 
 
 @task(ignore_result=True)
 @dbsession
 def get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit=100):
     log = get_logger(get_commits_stats)
     DBS = get_session()
     lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y,
                             ts_max_y)
     lockkey_path = config['app_conf']['cache_dir']
 
     log.info('running task with lockkey %s' % lockkey)
 
     try:
         lock = l = DaemonLock(file_=jn(lockkey_path, lockkey))
 
         # for js data compatibility cleans the key for person from '
         akc = lambda k: person(k).replace('"', "")
 
         co_day_auth_aggr = {}
         commits_by_day_aggregate = {}
         repo = Repository.get_by_repo_name(repo_name)
         if repo is None:
             return True
 
         repo = repo.scm_instance
         repo_size = repo.count()
         # return if repo have no revisions
         if repo_size < 1:
             lock.release()
             return True
 
         skip_date_limit = True
         parse_limit = int(config['app_conf'].get('commit_parse_limit'))
         last_rev = None
         last_cs = None
         timegetter = itemgetter('time')
 
         dbrepo = DBS.query(Repository)\
             .filter(Repository.repo_name == repo_name).scalar()
         cur_stats = DBS.query(Statistics)\
             .filter(Statistics.repository == dbrepo).scalar()
 
         if cur_stats is not None:
             last_rev = cur_stats.stat_on_revision
 
         if last_rev == repo.get_changeset().revision and repo_size > 1:
             # pass silently without any work if we're not on first revision or
             # current state of parsing revision(from db marker) is the
             # last revision
             lock.release()
             return True
 
         if cur_stats:
             commits_by_day_aggregate = OrderedDict(json.loads(
                                         cur_stats.commit_activity_combined))
             co_day_auth_aggr = json.loads(cur_stats.commit_activity)
 
         log.debug('starting parsing %s' % parse_limit)
         lmktime = mktime
 
         last_rev = last_rev + 1 if last_rev >= 0 else 0
         log.debug('Getting revisions from %s to %s' % (
              last_rev, last_rev + parse_limit)
         )
         for cs in repo[last_rev:last_rev + parse_limit]:
             log.debug('parsing %s' % cs)
             last_cs = cs  # remember last parsed changeset
             k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1],
                           cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0])
 
             if akc(cs.author) in co_day_auth_aggr:
                 try:
                     l = [timegetter(x) for x in
                          co_day_auth_aggr[akc(cs.author)]['data']]
                     time_pos = l.index(k)
                 except ValueError:
                     time_pos = None
 
                 if time_pos >= 0 and time_pos is not None:
 
                     datadict = \
                         co_day_auth_aggr[akc(cs.author)]['data'][time_pos]
 
                     datadict["commits"] += 1
                     datadict["added"] += len(cs.added)
                     datadict["changed"] += len(cs.changed)
                     datadict["removed"] += len(cs.removed)
 
                 else:
                     if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
 
                         datadict = {"time": k,
                                     "commits": 1,
                                     "added": len(cs.added),
                                     "changed": len(cs.changed),
                                     "removed": len(cs.removed),
                                    }
                         co_day_auth_aggr[akc(cs.author)]['data']\
                             .append(datadict)
 
             else:
                 if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
                     co_day_auth_aggr[akc(cs.author)] = {
                                         "label": akc(cs.author),
                                         "data": [{"time":k,
                                                  "commits":1,
                                                  "added":len(cs.added),
                                                  "changed":len(cs.changed),
                                                  "removed":len(cs.removed),
                                                  }],
                                         "schema": ["commits"],
                                         }
 
             #gather all data by day
             if k in commits_by_day_aggregate:
                 commits_by_day_aggregate[k] += 1
             else:
                 commits_by_day_aggregate[k] = 1
 
         overview_data = sorted(commits_by_day_aggregate.items(),
                                key=itemgetter(0))
 
         if not co_day_auth_aggr:
             co_day_auth_aggr[akc(repo.contact)] = {
                 "label": akc(repo.contact),
                 "data": [0, 1],
                 "schema": ["commits"],
             }
 
         stats = cur_stats if cur_stats else Statistics()
         stats.commit_activity = json.dumps(co_day_auth_aggr)
         stats.commit_activity_combined = json.dumps(overview_data)
 
         log.debug('last revison %s' % last_rev)
         leftovers = len(repo.revisions[last_rev:])
         log.debug('revisions to parse %s' % leftovers)
 
         if last_rev == 0 or leftovers < parse_limit:
             log.debug('getting code trending stats')
             stats.languages = json.dumps(__get_codes_stats(repo_name))
 
         try:
             stats.repository = dbrepo
             stats.stat_on_revision = last_cs.revision if last_cs else 0
             DBS.add(stats)
             DBS.commit()
         except:
             log.error(traceback.format_exc())
             DBS.rollback()
             lock.release()
             return False
 
         # final release
         lock.release()
 
         # execute another task if celery is enabled
         if len(repo.revisions) > 1 and CELERY_ON and recurse_limit > 0:
             recurse_limit -= 1
             run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y,
                      recurse_limit)
         if recurse_limit <= 0:
             log.debug('Breaking recursive mode due to reach of recurse limit')
         return True
     except LockHeld:
         log.info('LockHeld')
         return 'Task with key %s already running' % lockkey
 
 
 @task(ignore_result=True)
 @dbsession
-def send_email(recipients, subject, body='', html_body=''):
+def send_email(recipients, subject, body='', html_body='', headers=None):
     """
     Sends an email with defined parameters from the .ini files.
 
     :param recipients: list of recipients, if this is None, the defined email
         address from field 'email_to' and all admins is used instead
     :param subject: subject of the mail
     :param body: body of the mail
     :param html_body: html version of body
     """
     log = get_logger(send_email)
     DBS = get_session()
     assert isinstance(recipients, list), recipients
 
     email_config = config
     email_prefix = email_config.get('email_prefix', '')
     if email_prefix:
         subject = "%s %s" % (email_prefix, subject)
     if recipients is None:
         # if recipients are not defined we send to email_config + all admins
         admins = [u.email for u in User.query()
                   .filter(User.admin == True).all()]
         recipients = [email_config.get('email_to')] + admins
         log.warning("recipients not specified for '%s' - sending to admins %s", subject, ' '.join(recipients))
     elif not recipients:
         log.error("No recipients specified")
         return False
 
     mail_from = email_config.get('app_email_from', 'Kallithea')
     user = email_config.get('smtp_username')
     passwd = email_config.get('smtp_password')
     mail_server = email_config.get('smtp_server')
     mail_port = email_config.get('smtp_port')
     tls = str2bool(email_config.get('smtp_use_tls'))
     ssl = str2bool(email_config.get('smtp_use_ssl'))
     debug = str2bool(email_config.get('debug'))
     smtp_auth = email_config.get('smtp_auth')
 
     if not mail_server:
         log.error("SMTP mail server not configured - cannot send mail '%s' to %s", subject, ' '.join(recipients))
         log.warning("body:\n%s", body)
         log.warning("html:\n%s", html_body)
         return False
 
     try:
         m = SmtpMailer(mail_from, user, passwd, mail_server, smtp_auth,
                        mail_port, ssl, tls, debug=debug)
-        m.send(recipients, subject, body, html_body)
+        m.send(recipients, subject, body, html_body, headers=headers)
     except:
         log.error('Mail sending failed')
         log.error(traceback.format_exc())
         return False
     return True
 
 @task(ignore_result=False)
 @dbsession
 def create_repo(form_data, cur_user):
     from kallithea.model.repo import RepoModel
     from kallithea.model.user import UserModel
     from kallithea.model.db import Setting
 
     log = get_logger(create_repo)
     DBS = get_session()
 
     cur_user = UserModel(DBS)._get_user(cur_user)
 
     owner = cur_user
     repo_name = form_data['repo_name']
     repo_name_full = form_data['repo_name_full']
     repo_type = form_data['repo_type']
     description = form_data['repo_description']
     private = form_data['repo_private']
     clone_uri = form_data.get('clone_uri')
     repo_group = form_data['repo_group']
     landing_rev = form_data['repo_landing_rev']
     copy_fork_permissions = form_data.get('copy_permissions')
     copy_group_permissions = form_data.get('repo_copy_permissions')
     fork_of = form_data.get('fork_parent_id')
     state = form_data.get('repo_state', Repository.STATE_PENDING)
 
     # repo creation defaults, private and repo_type are filled in form
     defs = Setting.get_default_repo_settings(strip_prefix=True)
     enable_statistics = defs.get('repo_enable_statistics')
     enable_locking = defs.get('repo_enable_locking')
     enable_downloads = defs.get('repo_enable_downloads')
 
     try:
         repo = RepoModel(DBS)._create_repo(
             repo_name=repo_name_full,
             repo_type=repo_type,
             description=description,
             owner=owner,
             private=private,
             clone_uri=clone_uri,
             repo_group=repo_group,
             landing_rev=landing_rev,
             fork_of=fork_of,
             copy_fork_permissions=copy_fork_permissions,
             copy_group_permissions=copy_group_permissions,
             enable_statistics=enable_statistics,
             enable_locking=enable_locking,
             enable_downloads=enable_downloads,
             state=state
         )
 
         action_logger(cur_user, 'user_created_repo',
                       form_data['repo_name_full'], '', DBS)
 
         DBS.commit()
         # now create this repo on Filesystem
         RepoModel(DBS)._create_filesystem_repo(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_group=RepoModel(DBS)._get_repo_group(repo_group),
             clone_uri=clone_uri,
         )
         repo = Repository.get_by_repo_name(repo_name_full)
         log_create_repository(repo.get_dict(), created_by=owner.username)
 
         # update repo changeset caches initially
         repo.update_changeset_cache()
 
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
         DBS.commit()
     except Exception, e:
         log.warning('Exception %s occured when forking repository, '
                     'doing cleanup...' % e)
         # rollback things manually !
         repo = Repository.get_by_repo_name(repo_name_full)
         if repo:
             Repository.delete(repo.repo_id)
             DBS.commit()
             RepoModel(DBS)._delete_filesystem_repo(repo)
         raise
 
     # it's an odd fix to make celery fail task when exception occurs
     def on_failure(self, *args, **kwargs):
         pass
 
     return True
 
 
 @task(ignore_result=False)
 @dbsession
 def create_repo_fork(form_data, cur_user):
     """
     Creates a fork of repository using interval VCS methods
 
     :param form_data:
     :param cur_user:
     """
     from kallithea.model.repo import RepoModel
     from kallithea.model.user import UserModel
 
     log = get_logger(create_repo_fork)
     DBS = get_session()
 
     base_path = Repository.base_path()
     cur_user = UserModel(DBS)._get_user(cur_user)
 
     repo_name = form_data['repo_name']  # fork in this case
     repo_name_full = form_data['repo_name_full']
 
     repo_type = form_data['repo_type']
     owner = cur_user
     private = form_data['private']
     clone_uri = form_data.get('clone_uri')
     repo_group = form_data['repo_group']
     landing_rev = form_data['landing_rev']
     copy_fork_permissions = form_data.get('copy_permissions')
 
     try:
         fork_of = RepoModel(DBS)._get_repo(form_data.get('fork_parent_id'))
 
         fork_repo = RepoModel(DBS)._create_repo(
             repo_name=repo_name_full,
             repo_type=repo_type,
             description=form_data['description'],
             owner=owner,
             private=private,
             clone_uri=clone_uri,
             repo_group=repo_group,
             landing_rev=landing_rev,
             fork_of=fork_of,
             copy_fork_permissions=copy_fork_permissions
         )
         action_logger(cur_user, 'user_forked_repo:%s' % repo_name_full,
                       fork_of.repo_name, '', DBS)
         DBS.commit()
 
         update_after_clone = form_data['update_after_clone']
         source_repo_path = os.path.join(base_path, fork_of.repo_name)
 
         # now create this repo on Filesystem
         RepoModel(DBS)._create_filesystem_repo(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_group=RepoModel(DBS)._get_repo_group(repo_group),
             clone_uri=source_repo_path,
         )
         repo = Repository.get_by_repo_name(repo_name_full)
         log_create_repository(repo.get_dict(), created_by=owner.username)
 
         # update repo changeset caches initially
         repo.update_changeset_cache()
 
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
         DBS.commit()
     except Exception, e:
         log.warning('Exception %s occured when forking repository, '
                     'doing cleanup...' % e)
         #rollback things manually !
         repo = Repository.get_by_repo_name(repo_name_full)
         if repo:
             Repository.delete(repo.repo_id)
             DBS.commit()
             RepoModel(DBS)._delete_filesystem_repo(repo)
         raise
 
     # it's an odd fix to make celery fail task when exception occurs
     def on_failure(self, *args, **kwargs):
         pass
 
     return True
 
 
 def __get_codes_stats(repo_name):
     from kallithea.config.conf import LANGUAGES_EXTENSIONS_MAP
     repo = Repository.get_by_repo_name(repo_name).scm_instance
 
     tip = repo.get_changeset()
     code_stats = {}
 
     def aggregate(cs):
         for f in cs[2]:
             ext = lower(f.extension)
             if ext in LANGUAGES_EXTENSIONS_MAP.keys() and not f.is_binary:
                 if ext in code_stats:
                     code_stats[ext] += 1
                 else:
                     code_stats[ext] = 1
 
     map(aggregate, tip.walk('/'))
 
     return code_stats or {}

kallithea/lib/rcmail/smtp_mailer.py

@@ -1,105 +1,105 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.rcmail.smtp_mailer
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Simple smtp mailer used in Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Sep 13, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import time
 import logging
 import smtplib
 from socket import sslerror
 from email.utils import formatdate
 from kallithea.lib.rcmail.message import Message
 from kallithea.lib.rcmail.utils import DNS_NAME
 
 
 class SmtpMailer(object):
     """SMTP mailer class
 
     mailer = SmtpMailer(mail_from, user, passwd, mail_server, smtp_auth
                         mail_port, ssl, tls)
     mailer.send(recipients, subject, body, attachment_files)
 
     :param recipients might be a list of string or single string
     :param attachment_files is a dict of {filename:location}
         it tries to guess the mimetype and attach the file
 
     """
 
     def __init__(self, mail_from, user, passwd, mail_server, smtp_auth=None,
                  mail_port=None, ssl=False, tls=False, debug=False):
 
         self.mail_from = mail_from
         self.mail_server = mail_server
         self.mail_port = mail_port
         self.user = user
         self.passwd = passwd
         self.ssl = ssl
         self.tls = tls
         self.debug = debug
         self.auth = smtp_auth
 
     def send(self, recipients=[], subject='', body='', html='',
-             attachment_files=None):
+             attachment_files=None, headers=None):
 
         if isinstance(recipients, basestring):
             recipients = [recipients]
-        headers = {
-            'Date': formatdate(time.time())
-        }
+        if headers is None:
+            headers = {}
+        headers.setdefault('Date', formatdate(time.time()))
         msg = Message(subject, recipients, body, html, self.mail_from,
                       recipients_separator=", ", extra_headers=headers)
         raw_msg = msg.to_message()
 
         if self.ssl:
             smtp_serv = smtplib.SMTP_SSL(self.mail_server, self.mail_port,
                                          local_hostname=DNS_NAME.get_fqdn())
         else:
             smtp_serv = smtplib.SMTP(self.mail_server, self.mail_port,
                                      local_hostname=DNS_NAME.get_fqdn())
 
         if self.tls:
             smtp_serv.ehlo()
             smtp_serv.starttls()
 
         if self.debug:
             smtp_serv.set_debuglevel(1)
 
         smtp_serv.ehlo()
         if self.auth:
             smtp_serv.esmtp_features["auth"] = self.auth
 
         # if server requires authorization you must provide login and password
         # but only if we have them
         if self.user and self.passwd:
             smtp_serv.login(self.user, self.passwd)
 
         smtp_serv.sendmail(msg.sender, msg.send_to, raw_msg.as_string())
         logging.info('MAIL SENT TO: %s' % recipients)
 
         try:
             smtp_serv.quit()
         except sslerror:
             # sslerror is raised in tls connections on closing sometimes
             smtp_serv.close()

kallithea/model/comment.py

@@ -1,291 +1,301 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.model.comment
 ~~~~~~~~~~~~~~~~~~~~~~~
 
 comments model for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Nov 11, 2011
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import logging
 import traceback
 
 from pylons.i18n.translation import _
 from sqlalchemy.util.compat import defaultdict
 
 from kallithea.lib.utils2 import extract_mentioned_users, safe_unicode
 from kallithea.lib import helpers as h
 from kallithea.model import BaseModel
 from kallithea.model.db import ChangesetComment, User, Repository, \
     Notification, PullRequest
 from kallithea.model.notification import NotificationModel
 from kallithea.model.meta import Session
 
 log = logging.getLogger(__name__)
 
 
 class ChangesetCommentsModel(BaseModel):
 
     cls = ChangesetComment
 
     def __get_changeset_comment(self, changeset_comment):
         return self._get_instance(ChangesetComment, changeset_comment)
 
     def __get_pull_request(self, pull_request):
         return self._get_instance(PullRequest, pull_request)
 
     def _extract_mentions(self, s):
         user_objects = []
         for username in extract_mentioned_users(s):
             user_obj = User.get_by_username(username, case_insensitive=True)
             if user_obj:
                 user_objects.append(user_obj)
         return user_objects
 
     def _get_notification_data(self, repo, comment, user, comment_text,
                                line_no=None, revision=None, pull_request=None,
                                status_change=None, closing_pr=False):
         """
         Get notification data
 
         :param comment_text:
         :param line:
         :returns: tuple (subj,body,recipients,notification_type,email_kwargs)
         """
         # make notification
         body = comment_text  # text of the comment
         line = ''
         if line_no:
             line = _('on line %s') % line_no
 
         #changeset
         if revision:
             notification_type = Notification.TYPE_CHANGESET_COMMENT
             cs = repo.scm_instance.get_changeset(revision)
             desc = "%s" % (cs.short_id)
 
-            _url = h.url('changeset_home',
+            revision_url = h.url('changeset_home',
+                repo_name=repo.repo_name,
+                revision=revision,
+                qualified=True,)
+            comment_url = h.url('changeset_home',
                 repo_name=repo.repo_name,
                 revision=revision,
                 anchor='comment-%s' % comment.comment_id,
                 qualified=True,
             )
             subj = safe_unicode(
                 h.link_to('Re changeset: %(desc)s %(line)s' % \
                           {'desc': desc, 'line': line},
-                          _url)
+                          comment_url)
             )
             # get the current participants of this changeset
             recipients = ChangesetComment.get_users(revision=revision)
             # add changeset author if it's in kallithea system
             cs_author = User.get_from_cs_author(cs.author)
             if not cs_author:
                 #use repo owner if we cannot extract the author correctly
                 cs_author = repo.user
             recipients += [cs_author]
             email_kwargs = {
                 'status_change': status_change,
                 'cs_comment_user': h.person(user),
                 'cs_target_repo': h.url('summary_home', repo_name=repo.repo_name,
                                         qualified=True),
-                'cs_comment_url': _url,
+                'cs_comment_url': comment_url,
                 'raw_id': revision,
                 'message': cs.message,
                 'repo_name': repo.repo_name,
                 'short_id': h.short_id(revision),
                 'branch': cs.branch,
                 'comment_username': user.username,
+                'threading': [revision_url, comment_url], # TODO: url to line number
             }
         #pull request
         elif pull_request:
             notification_type = Notification.TYPE_PULL_REQUEST_COMMENT
             desc = comment.pull_request.title
             _org_ref_type, org_ref_name, _org_rev = comment.pull_request.org_ref.split(':')
-            _url = h.url('pullrequest_show',
+            pr_url = h.url('pullrequest_show',
+                repo_name=pull_request.other_repo.repo_name,
+                pull_request_id=pull_request.pull_request_id,
+                qualified=True,)
+            comment_url = h.url('pullrequest_show',
                 repo_name=pull_request.other_repo.repo_name,
                 pull_request_id=pull_request.pull_request_id,
                 anchor='comment-%s' % comment.comment_id,
                 qualified=True,
             )
             subj = safe_unicode(
                 h.link_to('Re pull request #%(pr_id)s: %(desc)s %(line)s' % \
                           {'desc': desc,
                            'pr_id': comment.pull_request.pull_request_id,
                            'line': line},
-                          _url)
+                          comment_url)
             )
             # get the current participants of this pull request
             recipients = ChangesetComment.get_users(pull_request_id=
                                                 pull_request.pull_request_id)
             # add pull request author
             recipients += [pull_request.author]
 
             # add the reviewers to notification
             recipients += [x.user for x in pull_request.reviewers]
 
             #set some variables for email notification
             email_kwargs = {
                 'pr_title': pull_request.title,
                 'pr_id': pull_request.pull_request_id,
                 'status_change': status_change,
                 'closing_pr': closing_pr,
-                'pr_comment_url': _url,
+                'pr_comment_url': comment_url,
                 'pr_comment_user': h.person(user),
                 'pr_target_repo': h.url('summary_home',
                                    repo_name=pull_request.other_repo.repo_name,
                                    qualified=True),
                 'repo_name': pull_request.other_repo.repo_name,
                 'ref': org_ref_name,
                 'comment_username': user.username,
+                'threading': [pr_url, comment_url], # TODO: url to line number
             }
 
         return subj, body, recipients, notification_type, email_kwargs
 
     def create(self, text, repo, user, revision=None, pull_request=None,
                f_path=None, line_no=None, status_change=None, closing_pr=False,
                send_email=True):
         """
         Creates new comment for changeset or pull request.
         If status_change is not None this comment is associated with a
         status change of changeset or changesets associated with pull request
 
         :param text:
         :param repo:
         :param user:
         :param revision:
         :param pull_request: (for emails, not for comments)
         :param f_path:
         :param line_no:
         :param status_change: (for emails, not for comments)
         :param closing_pr: (for emails, not for comments)
         :param send_email: also send email
         """
         if not text:
             log.warning('Missing text for comment, skipping...')
             return
 
         repo = self._get_repo(repo)
         user = self._get_user(user)
         comment = ChangesetComment()
         comment.repo = repo
         comment.author = user
         comment.text = text
         comment.f_path = f_path
         comment.line_no = line_no
 
         if revision:
             comment.revision = revision
         elif pull_request:
             pull_request = self.__get_pull_request(pull_request)
             comment.pull_request = pull_request
         else:
             raise Exception('Please specify revision or pull_request_id')
 
         Session().add(comment)
         Session().flush()
 
         if send_email:
             (subj, body, recipients, notification_type,
              email_kwargs) = self._get_notification_data(
                                 repo, comment, user,
                                 comment_text=text,
                                 line_no=line_no,
                                 revision=revision,
                                 pull_request=pull_request,
                                 status_change=status_change,
                                 closing_pr=closing_pr)
             # create notification objects, and emails
             NotificationModel().create(
                 created_by=user, subject=subj, body=body,
                 recipients=recipients, type_=notification_type,
                 email_kwargs=email_kwargs,
             )
 
             mention_recipients = set(self._extract_mentions(body))\
                                     .difference(recipients)
             if mention_recipients:
                 email_kwargs.update({'pr_mention': True})
                 subj = _('[Mention]') + ' ' + subj
                 NotificationModel().create(
                     created_by=user, subject=subj, body=body,
                     recipients=mention_recipients,
                     type_=notification_type,
                     email_kwargs=email_kwargs
                 )
 
         return comment
 
     def delete(self, comment):
         """
         Deletes given comment
 
         :param comment_id:
         """
         comment = self.__get_changeset_comment(comment)
         Session().delete(comment)
 
         return comment
 
     def get_comments(self, repo_id, revision=None, pull_request=None):
         """
         Gets main comments based on revision or pull_request_id
 
         :param repo_id:
         :param revision:
         :param pull_request:
         """
 
         q = ChangesetComment.query()\
                 .filter(ChangesetComment.repo_id == repo_id)\
                 .filter(ChangesetComment.line_no == None)\
                 .filter(ChangesetComment.f_path == None)
         if revision:
             q = q.filter(ChangesetComment.revision == revision)
         elif pull_request:
             pull_request = self.__get_pull_request(pull_request)
             q = q.filter(ChangesetComment.pull_request == pull_request)
         else:
             raise Exception('Please specify revision or pull_request')
         q = q.order_by(ChangesetComment.created_on)
         return q.all()
 
     def get_inline_comments(self, repo_id, revision=None, pull_request=None):
         q = Session().query(ChangesetComment)\
             .filter(ChangesetComment.repo_id == repo_id)\
             .filter(ChangesetComment.line_no != None)\
             .filter(ChangesetComment.f_path != None)\
             .order_by(ChangesetComment.comment_id.asc())\
 
         if revision:
             q = q.filter(ChangesetComment.revision == revision)
         elif pull_request:
             pull_request = self.__get_pull_request(pull_request)
             q = q.filter(ChangesetComment.pull_request == pull_request)
         else:
             raise Exception('Please specify revision or pull_request_id')
 
         comments = q.all()
 
         paths = defaultdict(lambda: defaultdict(list))
 
         for co in comments:
             paths[co.f_path][co.line_no].append(co)
         return paths.items()

kallithea/model/notification.py

@@ -1,316 +1,320 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.model.notification
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Model for notifications
 
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Nov 20, 2011
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 
 import os
 import logging
 import traceback
 
 from pylons import tmpl_context as c
 from pylons.i18n.translation import _
 
 import kallithea
 from kallithea.lib import helpers as h
 from kallithea.model import BaseModel
 from kallithea.model.db import Notification, User, UserNotification
 from kallithea.model.meta import Session
 
 log = logging.getLogger(__name__)
 
 
 class NotificationModel(BaseModel):
 
     cls = Notification
 
     def __get_notification(self, notification):
         if isinstance(notification, Notification):
             return notification
         elif isinstance(notification, (int, long)):
             return Notification.get(notification)
         else:
             if notification:
                 raise Exception('notification must be int, long or Instance'
                                 ' of Notification got %s' % type(notification))
 
     def create(self, created_by, subject, body, recipients=None,
                type_=Notification.TYPE_MESSAGE, with_email=True,
                email_kwargs={}):
         """
 
         Creates notification of given type
 
         :param created_by: int, str or User instance. User who created this
             notification
         :param subject:
         :param body:
         :param recipients: list of int, str or User objects, when None
             is given send to all admins
         :param type_: type of notification
         :param with_email: send email with this notification
         :param email_kwargs: additional dict to pass as args to email template
         """
         from kallithea.lib.celerylib import tasks, run_task
 
         if recipients and not getattr(recipients, '__iter__', False):
             raise Exception('recipients must be a list or iterable')
 
         created_by_obj = self._get_user(created_by)
 
         recipients_objs = []
         if recipients:
             for u in recipients:
                 obj = self._get_user(u)
                 if obj:
                     recipients_objs.append(obj)
                 else:
                     # TODO: inform user that requested operation couldn't be completed
                     log.error('cannot email unknown user %r', u)
             recipients_objs = set(recipients_objs)
             log.debug('sending notifications %s to %s' % (
                 type_, recipients_objs)
             )
         elif recipients is None:
             # empty recipients means to all admins
             recipients_objs = User.query().filter(User.admin == True).all()
             log.debug('sending notifications %s to admins: %s' % (
                 type_, recipients_objs)
             )
         #else: silently skip notification mails?
 
         # TODO: inform user who are notified
         notif = Notification.create(
             created_by=created_by_obj, subject=subject,
             body=body, recipients=recipients_objs, type_=type_
         )
 
         if not with_email:
             return notif
 
         #don't send email to person who created this comment
         rec_objs = set(recipients_objs).difference(set([created_by_obj]))
 
+        headers = None
+        if 'threading' in email_kwargs:
+            headers = {'References': ' '.join('<%s>' % x for x in email_kwargs['threading'])}
+
         # send email with notification to all other participants
         for rec in rec_objs:
             email_body = None  # we set body to none, we just send HTML emails
             ## this is passed into template
             kwargs = {'subject': subject,
                       'body': h.rst_w_mentions(body),
                       'when': h.fmt_date(notif.created_on),
                       'user': notif.created_by_user.username,
                       }
 
             kwargs.update(email_kwargs)
             email_subject = EmailNotificationModel()\
                                 .get_email_description(type_, **kwargs)
             email_body_html = EmailNotificationModel()\
                                 .get_email_tmpl(type_, **kwargs)
 
             run_task(tasks.send_email, [rec.email], email_subject, email_body,
-                     email_body_html)
+                     email_body_html, headers)
 
         return notif
 
     def delete(self, user, notification):
         # we don't want to remove actual notification just the assignment
         try:
             notification = self.__get_notification(notification)
             user = self._get_user(user)
             if notification and user:
                 obj = UserNotification.query()\
                         .filter(UserNotification.user == user)\
                         .filter(UserNotification.notification
                                 == notification)\
                         .one()
                 Session().delete(obj)
                 return True
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def get_for_user(self, user, filter_=None):
         """
         Get notifications for given user, filter them if filter dict is given
 
         :param user:
         :param filter:
         """
         user = self._get_user(user)
 
         q = UserNotification.query()\
             .filter(UserNotification.user == user)\
             .join((Notification, UserNotification.notification_id ==
                                  Notification.notification_id))
 
         if filter_:
             q = q.filter(Notification.type_.in_(filter_))
 
         return q.all()
 
     def mark_read(self, user, notification):
         try:
             notification = self.__get_notification(notification)
             user = self._get_user(user)
             if notification and user:
                 obj = UserNotification.query()\
                         .filter(UserNotification.user == user)\
                         .filter(UserNotification.notification
                                 == notification)\
                         .one()
                 obj.read = True
                 Session().add(obj)
                 return True
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def mark_all_read_for_user(self, user, filter_=None):
         user = self._get_user(user)
         q = UserNotification.query()\
             .filter(UserNotification.user == user)\
             .filter(UserNotification.read == False)\
             .join((Notification, UserNotification.notification_id ==
                                  Notification.notification_id))
         if filter_:
             q = q.filter(Notification.type_.in_(filter_))
 
         # this is a little inefficient but sqlalchemy doesn't support
         # update on joined tables :(
         for obj in q.all():
             obj.read = True
             Session().add(obj)
 
     def get_unread_cnt_for_user(self, user):
         user = self._get_user(user)
         return UserNotification.query()\
                 .filter(UserNotification.read == False)\
                 .filter(UserNotification.user == user).count()
 
     def get_unread_for_user(self, user):
         user = self._get_user(user)
         return [x.notification for x in UserNotification.query()\
                 .filter(UserNotification.read == False)\
                 .filter(UserNotification.user == user).all()]
 
     def get_user_notification(self, user, notification):
         user = self._get_user(user)
         notification = self.__get_notification(notification)
 
         return UserNotification.query()\
             .filter(UserNotification.notification == notification)\
             .filter(UserNotification.user == user).scalar()
 
     def make_description(self, notification, show_age=True):
         """
         Creates a human readable description based on properties
         of notification object
         """
         #alias
         _n = notification
         _map = {
             _n.TYPE_CHANGESET_COMMENT: _('%(user)s commented on changeset at %(when)s'),
             _n.TYPE_MESSAGE: _('%(user)s sent message at %(when)s'),
             _n.TYPE_MENTION: _('%(user)s mentioned you at %(when)s'),
             _n.TYPE_REGISTRATION: _('%(user)s registered in Kallithea at %(when)s'),
             _n.TYPE_PULL_REQUEST: _('%(user)s opened new pull request at %(when)s'),
             _n.TYPE_PULL_REQUEST_COMMENT: _('%(user)s commented on pull request at %(when)s')
         }
         tmpl = _map[notification.type_]
 
         if show_age:
             when = h.age(notification.created_on)
         else:
             when = h.fmt_date(notification.created_on)
 
         return tmpl % dict(
             user=notification.created_by_user.username,
             when=when,
             )
 
 
 class EmailNotificationModel(BaseModel):
 
     TYPE_CHANGESET_COMMENT = Notification.TYPE_CHANGESET_COMMENT
     TYPE_MESSAGE = Notification.TYPE_MESSAGE # only used for testing
     # Notification.TYPE_MENTION is not used
     TYPE_PASSWORD_RESET = 'password_link'
     TYPE_REGISTRATION = Notification.TYPE_REGISTRATION
     TYPE_PULL_REQUEST = Notification.TYPE_PULL_REQUEST
     TYPE_PULL_REQUEST_COMMENT = Notification.TYPE_PULL_REQUEST_COMMENT
     TYPE_DEFAULT = 'default'
 
     def __init__(self):
         super(EmailNotificationModel, self).__init__()
         self._template_root = kallithea.CONFIG['pylons.paths']['templates'][0]
         self._tmpl_lookup = kallithea.CONFIG['pylons.app_globals'].mako_lookup
         self.email_types = {
             self.TYPE_CHANGESET_COMMENT: 'email_templates/changeset_comment.html',
             self.TYPE_PASSWORD_RESET: 'email_templates/password_reset.html',
             self.TYPE_REGISTRATION: 'email_templates/registration.html',
             self.TYPE_DEFAULT: 'email_templates/default.html',
             self.TYPE_PULL_REQUEST: 'email_templates/pull_request.html',
             self.TYPE_PULL_REQUEST_COMMENT: 'email_templates/pull_request_comment.html',
         }
         self._subj_map = {
             self.TYPE_CHANGESET_COMMENT: _('Comment on %(repo_name)s changeset %(short_id)s on %(branch)s by %(comment_username)s'),
             self.TYPE_MESSAGE: 'Test Message',
             # self.TYPE_PASSWORD_RESET
             self.TYPE_REGISTRATION: _('New user %(new_username)s registered'),
             # self.TYPE_DEFAULT
             self.TYPE_PULL_REQUEST: _('Review request on %(repo_name)s pull request #%(pr_id)s from %(ref)s by %(pr_username)s'),
             self.TYPE_PULL_REQUEST_COMMENT: _('Comment on %(repo_name)s pull request #%(pr_id)s from %(ref)s by %(comment_username)s'),
         }
 
     def get_email_description(self, type_, **kwargs):
         """
         return subject for email based on given type
         """
         tmpl = self._subj_map[type_]
         try:
             subj = tmpl % kwargs
         except KeyError, e:
             log.error('error generating email subject for %r from %s: %s', type_, ','.join(self._subj_map.keys()), e)
             raise
         l = [str(x) for x in [kwargs.get('status_change'), kwargs.get('closing_pr') and _('Closing')] if x]
         if l:
             subj += ' (%s)' % (', '.join(l))
         return subj
 
     def get_email_tmpl(self, type_, **kwargs):
         """
         return generated template for email based on given type
         """
 
         base = self.email_types.get(type_, self.email_types[self.TYPE_DEFAULT])
         email_template = self._tmpl_lookup.get_template(base)
         # translator and helpers inject
         _kwargs = {'_': _,
                    'h': h,
                    'c': c}
         _kwargs.update(kwargs)
         log.debug('rendering tmpl %s with kwargs %s' % (base, _kwargs))
         return email_template.render(**_kwargs)

kallithea/model/pull_request.py

@@ -1,185 +1,186 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.model.pull_request
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 pull request model for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Jun 6, 2012
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import logging
 import datetime
 
 from pylons.i18n.translation import _
 
 from kallithea.model.meta import Session
 from kallithea.lib import helpers as h
 from kallithea.model import BaseModel
 from kallithea.model.db import PullRequest, PullRequestReviewers, Notification,\
     ChangesetStatus
 from kallithea.model.notification import NotificationModel
 from kallithea.lib.utils2 import safe_unicode
 
 
 log = logging.getLogger(__name__)
 
 
 class PullRequestModel(BaseModel):
 
     cls = PullRequest
 
     def __get_pull_request(self, pull_request):
         return self._get_instance(PullRequest, pull_request)
 
     def get_pullrequest_cnt_for_user(self, user):
         return PullRequest.query()\
                                 .join(PullRequestReviewers)\
                                 .filter(PullRequestReviewers.user_id == user)\
                                 .filter(PullRequest.status != PullRequest.STATUS_CLOSED)\
                                 .count()
 
     def get_all(self, repo_name, from_=False, closed=False):
         """Get all PRs for repo.
         Default is all PRs to the repo, PRs from the repo if from_.
         Closed PRs are only included if closed is true."""
         repo = self._get_repo(repo_name)
         q = PullRequest.query()
         if from_:
             q = q.filter(PullRequest.org_repo == repo)
         else:
             q = q.filter(PullRequest.other_repo == repo)
         if not closed:
             q = q.filter(PullRequest.status != PullRequest.STATUS_CLOSED)
         return q.order_by(PullRequest.created_on.desc()).all()
 
     def create(self, created_by, org_repo, org_ref, other_repo, other_ref,
                revisions, reviewers, title, description=None):
         from kallithea.model.changeset_status import ChangesetStatusModel
 
         created_by_user = self._get_user(created_by)
         org_repo = self._get_repo(org_repo)
         other_repo = self._get_repo(other_repo)
 
         new = PullRequest()
         new.org_repo = org_repo
         new.org_ref = org_ref
         new.other_repo = other_repo
         new.other_ref = other_ref
         new.revisions = revisions
         new.title = title
         new.description = description
         new.author = created_by_user
         Session().add(new)
         Session().flush()
 
         #reset state to under-review
         from kallithea.model.comment import ChangesetCommentsModel
         comment = ChangesetCommentsModel().create(
             text=u'Auto status change to %s' % (ChangesetStatus.get_status_lbl(ChangesetStatus.STATUS_UNDER_REVIEW)),
             repo=org_repo,
             user=new.author,
             pull_request=new,
             send_email=False
         )
         ChangesetStatusModel().set_status(
             org_repo,
             ChangesetStatus.STATUS_UNDER_REVIEW,
             new.author,
             comment,
             pull_request=new
         )
         self.__add_reviewers(new, reviewers)
         return new
 
     def __add_reviewers(self, pr, reviewers):
         #members
         for member in set(reviewers):
             _usr = self._get_user(member)
             reviewer = PullRequestReviewers(_usr, pr)
             Session().add(reviewer)
 
         revision_data = [(x.raw_id, x.message)
                          for x in map(pr.org_repo.get_changeset, pr.revisions)]
 
         #notification to reviewers
         pr_url = h.url('pullrequest_show', repo_name=pr.other_repo.repo_name,
                        pull_request_id=pr.pull_request_id,
                        qualified=True)
         subject = safe_unicode(
             h.link_to(
               _('%(user)s wants you to review pull request #%(pr_id)s: %(pr_title)s') % \
                 {'user': pr.author.username,
                  'pr_title': pr.title,
                  'pr_id': pr.pull_request_id},
                 pr_url)
             )
         body = pr.description
         _org_ref_type, org_ref_name, _org_rev = pr.org_ref.split(':')
         email_kwargs = {
             'pr_title': pr.title,
             'pr_user_created': h.person(pr.author),
             'pr_repo_url': h.url('summary_home', repo_name=pr.other_repo.repo_name,
                                  qualified=True,),
             'pr_url': pr_url,
             'pr_revisions': revision_data,
             'repo_name': pr.other_repo.repo_name,
             'pr_id': pr.pull_request_id,
             'ref': org_ref_name,
             'pr_username': pr.author.username,
+            'threading': [pr_url],
             }
         NotificationModel().create(created_by=pr.author, subject=subject, body=body,
                                    recipients=reviewers,
                                    type_=Notification.TYPE_PULL_REQUEST,
                                    email_kwargs=email_kwargs)
 
     def update_reviewers(self, pull_request, reviewers_ids):
         reviewers_ids = set(reviewers_ids)
         pull_request = self.__get_pull_request(pull_request)
         current_reviewers = PullRequestReviewers.query()\
                             .filter(PullRequestReviewers.pull_request==
                                    pull_request)\
                             .all()
         current_reviewers_ids = set([x.user.user_id for x in current_reviewers])
 
         to_add = reviewers_ids.difference(current_reviewers_ids)
         to_remove = current_reviewers_ids.difference(reviewers_ids)
 
         log.debug("Adding %s reviewers" % to_add)
         self.__add_reviewers(pull_request, to_add)
 
         log.debug("Removing %s reviewers" % to_remove)
         for uid in to_remove:
             reviewer = PullRequestReviewers.query()\
                     .filter(PullRequestReviewers.user_id==uid,
                             PullRequestReviewers.pull_request==pull_request)\
                     .scalar()
             if reviewer:
                 Session().delete(reviewer)
 
     def delete(self, pull_request):
         pull_request = self.__get_pull_request(pull_request)
         Session().delete(pull_request)
 
     def close_pull_request(self, pull_request):
         pull_request = self.__get_pull_request(pull_request)
         pull_request.status = PullRequest.STATUS_CLOSED
         pull_request.updated_on = datetime.datetime.now()
         Session().add(pull_request)