python3: removed from future imports

rhodecode/config/routing_links.py

@@ -1,106 +1,106 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Single source for redirection links.
 
 Goal of this module is to provide a single source of truth regarding external
 links. The data inside this module is used to configure the routing
 system of Enterprise and it is used also as a base to check if this data
 and our server configuration are in sync.
 
 .. py:data:: link_config
 
    Contains the configuration for external links. Each item is supposed to be
    a `dict` like this example::
 
       {"name": "url_name",
        "target": "https://rhodecode.com/r1/enterprise/keyword/",
        "external_target": "https://example.com/some-page.html",
       }
 
 then you can retrieve the url by simply calling the URL function:
 
 `h.route_path('url_name')`
 
 The redirection must be first implemented in our servers before
 you can see it working.
 """
 # pragma: no cover
-from __future__ import unicode_literals
+
 
 link_config = [
     {
         "name": "enterprise_docs",
         "target": "https://rhodecode.com/r1/enterprise/docs/",
         "external_target": "https://docs.rhodecode.com/RhodeCode-Enterprise/",
     },
     {
         "name": "enterprise_log_file_locations",
         "target": "https://rhodecode.com/r1/enterprise/docs/admin-system-overview/",
         "external_target": "https://docs.rhodecode.com/RhodeCode-Enterprise/admin/system-overview.html#log-files",
     },
     {
         "name": "enterprise_issue_tracker_settings",
         "target": "https://rhodecode.com/r1/enterprise/docs/issue-trackers-overview/",
         "external_target": "https://docs.rhodecode.com/RhodeCode-Enterprise/issue-trackers/issue-trackers.html",
     },
     {
         "name": "enterprise_svn_setup",
         "target": "https://rhodecode.com/r1/enterprise/docs/svn-setup/",
         "external_target": "https://docs.rhodecode.com/RhodeCode-Enterprise/admin/svn-http.html",
     },
     {
         "name": "enterprise_license_convert_from_old",
         "target": "https://rhodecode.com/r1/enterprise/convert-license/",
         "external_target": "https://rhodecode.com/u/license-upgrade",
     },
     {
         "name": "rst_help",
         "target": "http://docutils.sourceforge.io/docs/user/rst/quickref.html",
         "external_target": "https://docutils.sourceforge.io/docs/user/rst/quickref.html",
     },
     {
         "name": "markdown_help",
         "target": "https://daringfireball.net/projects/markdown/syntax",
         "external_target": "https://daringfireball.net/projects/markdown/syntax",
     },
     {
         "name": "rhodecode_official",
         "target": "https://rhodecode.com",
         "external_target": "https://rhodecode.com/",
     },
     {
         "name": "rhodecode_support",
         "target": "https://rhodecode.com/help/",
         "external_target": "https://rhodecode.com/support",
     },
     {
         "name": "rhodecode_translations",
         "target": "https://rhodecode.com/translate/enterprise",
         "external_target": "https://explore.transifex.com/rhodecode/RhodeCode/",
     },
 
 ]
 
 
 def connect_redirection_links(config):
     for link in link_config:
         config.add_route(link['name'], link['target'], static=True)

rhodecode/integrations/types/email.py

@@ -1,329 +1,329 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-from __future__ import unicode_literals
+
 import logging
 
 import colander
 import deform.widget
 from mako.template import Template
 
 from rhodecode import events
 from rhodecode.model.validation_schema.widgets import CheckboxChoiceWidgetDesc
 from rhodecode.translation import _
 from rhodecode.lib.celerylib import run_task
 from rhodecode.lib.celerylib import tasks
 from rhodecode.integrations.types.base import (
     IntegrationTypeBase, render_with_traceback)
 
 
 log = logging.getLogger(__name__)
 
 REPO_PUSH_TEMPLATE_PLAINTEXT = Template('''
 Commits:
 
 % for commit in data['push']['commits']:
 ${commit['url']} by ${commit['author']} at ${commit['date']}
 ${commit['message']}
 ----
 
 % endfor
 ''')
 
 REPO_PUSH_TEMPLATE_HTML = Template('''
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
     <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
     <title>${subject}</title>
     <style type="text/css">
         /* Based on The MailChimp Reset INLINE: Yes. */
         #outlook a {padding:0;} /* Force Outlook to provide a "view in browser" menu link. */
         body{width:100% !important; -webkit-text-size-adjust:100%; -ms-text-size-adjust:100%; margin:0; padding:0;}
         /* Prevent Webkit and Windows Mobile platforms from changing default font sizes.*/
         .ExternalClass {width:100%;} /* Force Hotmail to display emails at full width */
         .ExternalClass, .ExternalClass p, .ExternalClass span, .ExternalClass font, .ExternalClass td, .ExternalClass div {line-height: 100%;}
         /* Forces Hotmail to display normal line spacing.  More on that: http://www.emailonacid.com/forum/viewthread/43/ */
         #backgroundTable {margin:0; padding:0; line-height: 100% !important;}
         /* End reset */
 
         /* defaults for images*/
         img {outline:none; text-decoration:none; -ms-interpolation-mode: bicubic;}
         a img {border:none;}
         .image_fix {display:block;}
 
         body {line-height:1.2em;}
         p {margin: 0 0 20px;}
         h1, h2, h3, h4, h5, h6 {color:#323232!important;}
         a {color:#427cc9;text-decoration:none;outline:none;cursor:pointer;}
         a:focus {outline:none;}
         a:hover {color: #305b91;}
         h1 a, h2 a, h3 a, h4 a, h5 a, h6 a {color:#427cc9!important;text-decoration:none!important;}
         h1 a:active, h2 a:active,  h3 a:active, h4 a:active, h5 a:active, h6 a:active {color: #305b91!important;}
         h1 a:visited, h2 a:visited,  h3 a:visited, h4 a:visited, h5 a:visited, h6 a:visited {color: #305b91!important;}
         table {font-size:13px;border-collapse:collapse;mso-table-lspace:0pt;mso-table-rspace:0pt;}
         table td {padding:.65em 1em .65em 0;border-collapse:collapse;vertical-align:top;text-align:left;}
         input {display:inline;border-radius:2px;border-style:solid;border: 1px solid #dbd9da;padding:.5em;}
         input:focus {outline: 1px solid #979797}
         @media only screen and (-webkit-min-device-pixel-ratio: 2) {
         /* Put your iPhone 4g styles in here */
         }
 
         /* Android targeting */
         @media only screen and (-webkit-device-pixel-ratio:.75){
         /* Put CSS for low density (ldpi) Android layouts in here */
         }
         @media only screen and (-webkit-device-pixel-ratio:1){
         /* Put CSS for medium density (mdpi) Android layouts in here */
         }
         @media only screen and (-webkit-device-pixel-ratio:1.5){
         /* Put CSS for high density (hdpi) Android layouts in here */
         }
         /* end Android targeting */
 
     </style>
 
     <!-- Targeting Windows Mobile -->
     <!--[if IEMobile 7]>
     <style type="text/css">
 
     </style>
     <![endif]-->
 
     <!--[if gte mso 9]>
         <style>
         /* Target Outlook 2007 and 2010 */
         </style>
     <![endif]-->
 </head>
 <body>
 <!-- Wrapper/Container Table: Use a wrapper table to control the width and the background color consistently of your email. Use this approach instead of setting attributes on the body tag. -->
 <table cellpadding="0" cellspacing="0" border="0" id="backgroundTable" align="left" style="margin:1%;width:97%;padding:0;font-family:sans-serif;font-weight:100;border:1px solid #dbd9da">
     <tr>
         <td valign="top" style="padding:0;">
             <table cellpadding="0" cellspacing="0" border="0" align="left" width="100%">
                 <tr><td style="width:100%;padding:7px;background-color:#202020" valign="top">
                     <a style="color:#eeeeee;text-decoration:none;" href="${instance_url}">
                         ${'RhodeCode'}
                     </a>
                 </td></tr>
                 <tr>
                     <td style="padding:15px;" valign="top">
                         % if data['push']['commits']:
                             % for commit in data['push']['commits']:
                             <a href="${commit['url']}">${commit['short_id']}</a> by ${commit['author']} at ${commit['date']} <br/>
                             ${commit['message_html']} <br/>
                             <br/>
                             % endfor
                         % else:
                             No commit data
                         % endif
                     </td>
                 </tr>
             </table>
         </td>
     </tr>
 </table>
 <!-- End of wrapper table -->
 <p><a style="margin-top:15px;margin-left:1%;font-family:sans-serif;font-weight:100;font-size:11px;color:#666666;text-decoration:none;" href="${instance_url}">
     ${'This is a notification from RhodeCode. %(instance_url)s' % {'instance_url': instance_url}}
 </a></p>
 </body>
 </html>
 ''')
 
 
 class EmailSettingsSchema(colander.Schema):
     @colander.instantiate(validator=colander.Length(min=1))
     class recipients(colander.SequenceSchema):
         title = _('Recipients')
         description = _('Email addresses to send push events to')
         widget = deform.widget.SequenceWidget(min_len=1)
 
         recipient = colander.SchemaNode(
             colander.String(),
             title=_('Email address'),
             description=_('Email address'),
             default='',
             validator=colander.Email(),
             widget=deform.widget.TextInputWidget(
                 placeholder='[e-mail]',
             ),
         )
 
 
 class EmailIntegrationType(IntegrationTypeBase):
     key = 'email'
     display_name = _('Email')
     description = _('Send repo push summaries to a list of recipients via email')
 
     valid_events = [
         events.RepoPushEvent
     ]
 
     @classmethod
     def icon(cls):
         return '''
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <svg
    xmlns:dc="http://purl.org/dc/elements/1.1/"
    xmlns:cc="http://creativecommons.org/ns#"
    xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
    xmlns:svg="http://www.w3.org/2000/svg"
    xmlns="http://www.w3.org/2000/svg"
    xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
    xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
    viewBox="0 -256 1850 1850"
    id="svg2989"
    version="1.1"
    inkscape:version="0.48.3.1 r9886"
    width="100%"
    height="100%"
    sodipodi:docname="envelope_font_awesome.svg">
   <metadata
      id="metadata2999">
     <rdf:RDF>
       <cc:Work
          rdf:about="">
         <dc:format>image/svg+xml</dc:format>
         <dc:type
            rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
       </cc:Work>
     </rdf:RDF>
   </metadata>
   <defs
      id="defs2997" />
   <sodipodi:namedview
      pagecolor="#ffffff"
      bordercolor="#666666"
      borderopacity="1"
      objecttolerance="10"
      gridtolerance="10"
      guidetolerance="10"
      inkscape:pageopacity="0"
      inkscape:pageshadow="2"
      inkscape:window-width="640"
      inkscape:window-height="480"
      id="namedview2995"
      showgrid="false"
      inkscape:zoom="0.13169643"
      inkscape:cx="896"
      inkscape:cy="896"
      inkscape:window-x="0"
      inkscape:window-y="25"
      inkscape:window-maximized="0"
      inkscape:current-layer="svg2989" />
   <g
      transform="matrix(1,0,0,-1,37.966102,1282.678)"
      id="g2991">
     <path
        d="m 1664,32 v 768 q -32,-36 -69,-66 -268,-206 -426,-338 -51,-43 -83,-67 -32,-24 -86.5,-48.5 Q 945,256 897,256 h -1 -1 Q 847,256 792.5,280.5 738,305 706,329 674,353 623,396 465,528 197,734 160,764 128,800 V 32 Q 128,19 137.5,9.5 147,0 160,0 h 1472 q 13,0 22.5,9.5 9.5,9.5 9.5,22.5 z m 0,1051 v 11 13.5 q 0,0 -0.5,13 -0.5,13 -3,12.5 -2.5,-0.5 -5.5,9 -3,9.5 -9,7.5 -6,-2 -14,2.5 H 160 q -13,0 -22.5,-9.5 Q 128,1133 128,1120 128,952 275,836 468,684 676,519 682,514 711,489.5 740,465 757,452 774,439 801.5,420.5 829,402 852,393 q 23,-9 43,-9 h 1 1 q 20,0 43,9 23,9 50.5,27.5 27.5,18.5 44.5,31.5 17,13 46,37.5 29,24.5 35,29.5 208,165 401,317 54,43 100.5,115.5 46.5,72.5 46.5,131.5 z m 128,37 V 32 q 0,-66 -47,-113 -47,-47 -113,-47 H 160 Q 94,-128 47,-81 0,-34 0,32 v 1088 q 0,66 47,113 47,47 113,47 h 1472 q 66,0 113,-47 47,-47 47,-113 z"
        id="path2993"
        inkscape:connector-curvature="0"
        style="fill:currentColor" />
   </g>
 </svg>
 '''
 
     def settings_schema(self):
         schema = EmailSettingsSchema()
         schema.add(colander.SchemaNode(
             colander.Set(),
             widget=CheckboxChoiceWidgetDesc(
                 values=sorted(
                     [(e.name, e.display_name, e.description) for e in self.valid_events]
                 ),
             ),
             description="List of events activated for this integration",
             name='events'
         ))
         return schema
 
     def send_event(self, event):
         log.debug('handling event %s with integration %s', event.name, self)
 
         if event.__class__ not in self.valid_events:
             log.debug('event %r not present in valid event list (%s)', event, self.valid_events)
             return
 
         if not self.event_enabled(event):
             # NOTE(marcink): for legacy reasons we're skipping this check...
             # since the email event haven't had any settings...
             pass
 
         handler = EmailEventHandler(self.settings)
         handler(event, event_data=event.as_dict())
 
 
 class EmailEventHandler(object):
     def __init__(self, integration_settings):
         self.integration_settings = integration_settings
 
     def __call__(self, event, event_data):
         if isinstance(event, events.RepoPushEvent):
             self.repo_push_handler(event, event_data)
         else:
             log.debug('ignoring event: %r', event)
 
     def repo_push_handler(self, event, data):
         commit_num = len(data['push']['commits'])
         server_url = data['server_url']
 
         if commit_num == 1:
             if data['push']['branches']:
                 _subject = '[{repo_name}] {author} pushed {commit_num} commit on branches: {branches}'
             else:
                 _subject = '[{repo_name}] {author} pushed {commit_num} commit'
             subject = _subject.format(
                 author=data['actor']['username'],
                 repo_name=data['repo']['repo_name'],
                 commit_num=commit_num,
                 branches=', '.join(
                     branch['name'] for branch in data['push']['branches'])
             )
         else:
             if data['push']['branches']:
                 _subject = '[{repo_name}] {author} pushed {commit_num} commits on branches: {branches}'
             else:
                 _subject = '[{repo_name}] {author} pushed {commit_num} commits'
             subject = _subject.format(
                 author=data['actor']['username'],
                 repo_name=data['repo']['repo_name'],
                 commit_num=commit_num,
                 branches=', '.join(
                     branch['name'] for branch in data['push']['branches']))
 
         email_body_plaintext = render_with_traceback(
             REPO_PUSH_TEMPLATE_PLAINTEXT,
             data=data,
             subject=subject,
             instance_url=server_url)
 
         email_body_html = render_with_traceback(
             REPO_PUSH_TEMPLATE_HTML,
             data=data,
             subject=subject,
             instance_url=server_url)
 
         recipients = self.integration_settings['recipients']
         for email_address in recipients:
             run_task(tasks.send_email, email_address, subject,
                      email_body_plaintext, email_body_html)

rhodecode/integrations/types/hipchat.py

@@ -1,253 +1,253 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-from __future__ import unicode_literals
+
 import deform
 import logging
 import requests
 import colander
 import textwrap
 from mako.template import Template
 from rhodecode import events
 from rhodecode.model.validation_schema.widgets import CheckboxChoiceWidgetDesc
 from rhodecode.translation import _
 from rhodecode.lib import helpers as h
 from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask
 from rhodecode.lib.colander_utils import strip_whitespace
 from rhodecode.integrations.types.base import (
     IntegrationTypeBase, CommitParsingDataHandler, render_with_traceback,
     requests_retry_call)
 
 log = logging.getLogger(__name__)
 
 REPO_PUSH_TEMPLATE = Template('''
 <b>${data['actor']['username']}</b> pushed to repo <a href="${data['repo']['url']}">${data['repo']['repo_name']}</a>:
 <br>
 <ul>
 %for branch, branch_commits in branches_commits.items():
     <li>
         % if branch:
         <a href="${branch_commits['branch']['url']}">branch: ${branch_commits['branch']['name']}</a>
         % else:
         to trunk
         % endif
         <ul>
         % for commit in branch_commits['commits']:
             <li><a href="${commit['url']}">${commit['short_id']}</a> - ${commit['message_html']}</li>
         % endfor
         </ul>
     </li>
 %endfor
 ''')
 
 
 class HipchatSettingsSchema(colander.Schema):
     color_choices = [
         ('yellow', _('Yellow')),
         ('red', _('Red')),
         ('green', _('Green')),
         ('purple', _('Purple')),
         ('gray', _('Gray')),
     ]
 
     server_url = colander.SchemaNode(
         colander.String(),
         title=_('Hipchat server URL'),
         description=_('Hipchat integration url.'),
         default='',
         preparer=strip_whitespace,
         validator=colander.url,
         widget=deform.widget.TextInputWidget(
             placeholder='https://?.hipchat.com/v2/room/?/notification?auth_token=?',
         ),
     )
     notify = colander.SchemaNode(
         colander.Bool(),
         title=_('Notify'),
         description=_('Make a notification to the users in room.'),
         missing=False,
         default=False,
     )
     color = colander.SchemaNode(
         colander.String(),
         title=_('Color'),
         description=_('Background color of message.'),
         missing='',
         validator=colander.OneOf([x[0] for x in color_choices]),
         widget=deform.widget.Select2Widget(
             values=color_choices,
         ),
     )
 
 
 class HipchatIntegrationType(IntegrationTypeBase, CommitParsingDataHandler):
     key = 'hipchat'
     display_name = _('Hipchat')
     description = _('Send events such as repo pushes and pull requests to '
                     'your hipchat channel.')
 
     @classmethod
     def icon(cls):
         return '''<?xml version="1.0" encoding="utf-8"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 1000 1000" enable-background="new 0 0 1000 1000" xml:space="preserve"><g><g transform="translate(0.000000,511.000000) scale(0.100000,-0.100000)"><path fill="#205281" d="M4197.1,4662.4c-1661.5-260.4-3018-1171.6-3682.6-2473.3C219.9,1613.6,100,1120.3,100,462.6c0-1014,376.8-1918.4,1127-2699.4C2326.7-3377.6,3878.5-3898.3,5701-3730.5l486.5,44.5l208.9-123.3c637.2-373.4,1551.8-640.6,2240.4-650.9c304.9-6.9,335.7,0,417.9,75.4c185,174.7,147.3,411.1-89.1,548.1c-315.2,181.6-620,544.7-733.1,870.1l-51.4,157.6l472.7,472.7c349.4,349.4,520.7,551.5,657.7,774.2c784.5,1281.2,784.5,2788.5,0,4052.6c-236.4,376.8-794.8,966-1178.4,1236.7c-572.1,407.7-1264.1,709.1-1993.7,870.1c-267.2,58.2-479.6,75.4-1038,82.2C4714.4,4686.4,4310.2,4679.6,4197.1,4662.4z M5947.6,3740.9c1856.7-380.3,3127.6-1709.4,3127.6-3275c0-1000.3-534.4-1949.2-1466.2-2600.1c-188.4-133.6-287.8-226.1-301.5-284.4c-41.1-157.6,263.8-938.6,397.4-1020.8c20.5-10.3,34.3-44.5,34.3-75.4c0-167.8-811.9,195.3-1363.4,609.8l-181.6,137l-332.3-58.2c-445.3-78.8-1281.2-78.8-1702.6,0C2796-2569.2,1734.1-1832.6,1220.2-801.5C983.8-318.5,905,51.5,929,613.3c27.4,640.6,243.2,1192.1,685.1,1740.3c620,770.8,1661.5,1305.2,2822.8,1452.5C4806.9,3854,5553.7,3819.7,5947.6,3740.9z"/><path fill="#205281" d="M2381.5-345.9c-75.4-106.2-68.5-167.8,34.3-322c332.3-500.2,1010.6-928.4,1760.8-1120.2c417.9-106.2,1226.4-106.2,1644.3,0c712.5,181.6,1270.9,517.3,1685.4,1014C7681-561.7,7715.3-424.7,7616-325.4c-89.1,89.1-167.9,65.1-431.7-133.6c-835.8-630.3-2028-856.4-3086.5-585.8C3683.3-938.6,3142-685,2830.3-448.7C2576.8-253.4,2463.7-229.4,2381.5-345.9z"/></g></g><!-- Svg Vector Icons : http://www.onlinewebfonts.com/icon --></svg>'''
 
     valid_events = [
         events.PullRequestCloseEvent,
         events.PullRequestMergeEvent,
         events.PullRequestUpdateEvent,
         events.PullRequestCommentEvent,
         events.PullRequestReviewEvent,
         events.PullRequestCreateEvent,
         events.RepoPushEvent,
         events.RepoCreateEvent,
     ]
 
     def send_event(self, event):
         if event.__class__ not in self.valid_events:
             log.debug('event %r not present in valid event list (%s)', event, self.valid_events)
             return
 
         if not self.event_enabled(event):
             return
 
         data = event.as_dict()
 
         text = '<b>%s<b> caused a <b>%s</b> event' % (
             data['actor']['username'], event.name)
 
         if isinstance(event, events.PullRequestCommentEvent):
             text = self.format_pull_request_comment_event(event, data)
         elif isinstance(event, events.PullRequestCommentEditEvent):
             text = self.format_pull_request_comment_event(event, data)
         elif isinstance(event, events.PullRequestReviewEvent):
             text = self.format_pull_request_review_event(event, data)
         elif isinstance(event, events.PullRequestEvent):
             text = self.format_pull_request_event(event, data)
         elif isinstance(event, events.RepoPushEvent):
             text = self.format_repo_push_event(data)
         elif isinstance(event, events.RepoCreateEvent):
             text = self.format_repo_create_event(data)
         else:
             log.error('unhandled event type: %r', event)
 
         run_task(post_text_to_hipchat, self.settings, text)
 
     def settings_schema(self):
         schema = HipchatSettingsSchema()
         schema.add(colander.SchemaNode(
             colander.Set(),
             widget=CheckboxChoiceWidgetDesc(
                 values=sorted(
                     [(e.name, e.display_name, e.description) for e in self.valid_events]
                 ),
             ),
             description="List of events activated for this integration",
             name='events'
         ))
 
         return schema
 
     def format_pull_request_comment_event(self, event, data):
         comment_text = data['comment']['text']
         if len(comment_text) > 200:
             comment_text = '{comment_text}<a href="{comment_url}">...<a/>'.format(
                 comment_text=h.html_escape(comment_text[:200]),
                 comment_url=data['comment']['url'],
             )
 
         comment_status = ''
         if data['comment']['status']:
             comment_status = '[{}]: '.format(data['comment']['status'])
 
         return (textwrap.dedent(
             '''
             {user} commented on pull request <a href="{pr_url}">{number}</a> - {pr_title}:
             >>> {comment_status}{comment_text}
             ''').format(
                 comment_status=comment_status,
                 user=data['actor']['username'],
                 number=data['pullrequest']['pull_request_id'],
                 pr_url=data['pullrequest']['url'],
                 pr_status=data['pullrequest']['status'],
                 pr_title=h.html_escape(data['pullrequest']['title']),
                 comment_text=h.html_escape(comment_text)
             )
         )
 
     def format_pull_request_review_event(self, event, data):
         return (textwrap.dedent(
             '''
             Status changed to {pr_status} for pull request <a href="{pr_url}">#{number}</a> - {pr_title}
             ''').format(
                 user=data['actor']['username'],
                 number=data['pullrequest']['pull_request_id'],
                 pr_url=data['pullrequest']['url'],
                 pr_status=data['pullrequest']['status'],
                 pr_title=h.html_escape(data['pullrequest']['title']),
             )
         )
 
     def format_pull_request_event(self, event, data):
         action = {
             events.PullRequestCloseEvent: 'closed',
             events.PullRequestMergeEvent: 'merged',
             events.PullRequestUpdateEvent: 'updated',
             events.PullRequestCreateEvent: 'created',
         }.get(event.__class__, str(event.__class__))
 
         return ('Pull request <a href="{url}">#{number}</a> - {title} '
                 '{action} by <b>{user}</b>').format(
             user=data['actor']['username'],
             number=data['pullrequest']['pull_request_id'],
             url=data['pullrequest']['url'],
             title=h.html_escape(data['pullrequest']['title']),
             action=action
         )
 
     def format_repo_push_event(self, data):
         branches_commits = self.aggregate_branch_data(
             data['push']['branches'], data['push']['commits'])
 
         result = render_with_traceback(
             REPO_PUSH_TEMPLATE,
             data=data,
             branches_commits=branches_commits,
         )
         return result
 
     def format_repo_create_event(self, data):
         return '<a href="{}">{}</a> ({}) repository created by <b>{}</b>'.format(
             data['repo']['url'],
             h.html_escape(data['repo']['repo_name']),
             data['repo']['repo_type'],
             data['actor']['username'],
         )
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def post_text_to_hipchat(settings, text):
     log.debug('sending %s to hipchat %s', text, settings['server_url'])
     json_message = {
         "message": text,
         "color": settings.get('color', 'yellow'),
         "notify": settings.get('notify', False),
     }
     req_session = requests_retry_call()
     resp = req_session.post(settings['server_url'], json=json_message, timeout=60)
     resp.raise_for_status()  # raise exception on a failed request

rhodecode/integrations/types/slack.py

@@ -1,354 +1,354 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-from __future__ import unicode_literals
+
 import re
 import time
 import textwrap
 import logging
 
 import deform
 import requests
 import colander
 from mako.template import Template
 
 from rhodecode import events
 from rhodecode.model.validation_schema.widgets import CheckboxChoiceWidgetDesc
 from rhodecode.translation import _
 from rhodecode.lib import helpers as h
 from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask
 from rhodecode.lib.colander_utils import strip_whitespace
 from rhodecode.integrations.types.base import (
     IntegrationTypeBase, CommitParsingDataHandler, render_with_traceback,
     requests_retry_call)
 
 log = logging.getLogger(__name__)
 
 
 def html_to_slack_links(message):
     return re.compile(r'<a .*?href=["\'](.+?)".*?>(.+?)</a>').sub(
         r'<\1|\2>', message)
 
 
 REPO_PUSH_TEMPLATE = Template('''
 <%
     def branch_text(branch):
         if branch:
             return 'on branch: <{}|{}>'.format(branch_commits['branch']['url'], branch_commits['branch']['name'])
         else:
             ## case for SVN no branch push...
             return 'to trunk'
 %> \
 
 % for branch, branch_commits in branches_commits.items():
 ${len(branch_commits['commits'])} ${'commit' if len(branch_commits['commits']) == 1 else 'commits'} ${branch_text(branch)}
 % for commit in branch_commits['commits']:
 `<${commit['url']}|${commit['short_id']}>` - ${commit['message_html']|html_to_slack_links}
 % endfor
 % endfor
 ''')
 
 
 class SlackSettingsSchema(colander.Schema):
     service = colander.SchemaNode(
         colander.String(),
         title=_('Slack service URL'),
         description=h.literal(_(
             'This can be setup at the '
             '<a href="https://my.slack.com/services/new/incoming-webhook/">'
             'slack app manager</a>')),
         default='',
         preparer=strip_whitespace,
         validator=colander.url,
         widget=deform.widget.TextInputWidget(
             placeholder='https://hooks.slack.com/services/...',
         ),
     )
     username = colander.SchemaNode(
         colander.String(),
         title=_('Username'),
         description=_('Username to show notifications coming from.'),
         missing='Rhodecode',
         preparer=strip_whitespace,
         widget=deform.widget.TextInputWidget(
             placeholder='Rhodecode'
         ),
     )
     channel = colander.SchemaNode(
         colander.String(),
         title=_('Channel'),
         description=_('Channel to send notifications to.'),
         missing='',
         preparer=strip_whitespace,
         widget=deform.widget.TextInputWidget(
             placeholder='#general'
         ),
     )
     icon_emoji = colander.SchemaNode(
         colander.String(),
         title=_('Emoji'),
         description=_('Emoji to use eg. :studio_microphone:'),
         missing='',
         preparer=strip_whitespace,
         widget=deform.widget.TextInputWidget(
             placeholder=':studio_microphone:'
         ),
     )
 
 
 class SlackIntegrationType(IntegrationTypeBase, CommitParsingDataHandler):
     key = 'slack'
     display_name = _('Slack')
     description = _('Send events such as repo pushes and pull requests to '
                     'your slack channel.')
 
     @classmethod
     def icon(cls):
         return '''<?xml version="1.0" encoding="UTF-8" standalone="no"?><svg viewBox="0 0 256 256" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid"><g><path d="M165.963541,15.8384262 C162.07318,3.86308197 149.212328,-2.69009836 137.239082,1.20236066 C125.263738,5.09272131 118.710557,17.9535738 122.603016,29.9268197 L181.550164,211.292328 C185.597902,222.478689 197.682361,228.765377 209.282098,225.426885 C221.381246,221.943607 228.756984,209.093246 224.896,197.21023 C224.749115,196.756984 165.963541,15.8384262 165.963541,15.8384262" fill="#DFA22F"></path><path d="M74.6260984,45.515541 C70.7336393,33.5422951 57.8727869,26.9891148 45.899541,30.8794754 C33.9241967,34.7698361 27.3710164,47.6306885 31.2634754,59.6060328 L90.210623,240.971541 C94.2583607,252.157902 106.34282,258.44459 117.942557,255.104 C130.041705,251.62282 137.417443,238.772459 133.556459,226.887344 C133.409574,226.436197 74.6260984,45.515541 74.6260984,45.515541" fill="#3CB187"></path><path d="M240.161574,166.045377 C252.136918,162.155016 258.688,149.294164 254.797639,137.31882 C250.907279,125.345574 238.046426,118.792393 226.07318,122.682754 L44.7076721,181.632 C33.5213115,185.677639 27.234623,197.762098 30.5731148,209.361836 C34.0563934,221.460984 46.9067541,228.836721 58.7897705,224.975738 C59.2430164,224.828852 240.161574,166.045377 240.161574,166.045377" fill="#CE1E5B"></path><path d="M82.507541,217.270557 C94.312918,213.434754 109.528131,208.491016 125.855475,203.186361 C122.019672,191.380984 117.075934,176.163672 111.76918,159.83423 L68.4191475,173.924721 L82.507541,217.270557" fill="#392538"></path><path d="M173.847082,187.591344 C190.235279,182.267803 205.467279,177.31777 217.195016,173.507148 C213.359213,161.70177 208.413377,146.480262 203.106623,130.146623 L159.75659,144.237115 L173.847082,187.591344" fill="#BB242A"></path><path d="M210.484459,74.7058361 C222.457705,70.8154754 229.010885,57.954623 225.120525,45.9792787 C221.230164,34.0060328 208.369311,27.4528525 196.393967,31.3432131 L15.028459,90.292459 C3.84209836,94.3380984 -2.44459016,106.422557 0.896,118.022295 C4.37718033,130.121443 17.227541,137.49718 29.1126557,133.636197 C29.5638033,133.489311 210.484459,74.7058361 210.484459,74.7058361" fill="#72C5CD"></path><path d="M52.8220328,125.933115 C64.6274098,122.097311 79.8468197,117.151475 96.1762623,111.84682 C90.8527213,95.4565246 85.9026885,80.2245246 82.0920656,68.4946885 L38.731541,82.5872787 L52.8220328,125.933115" fill="#248C73"></path><path d="M144.159475,96.256 C160.551869,90.9303607 175.785967,85.9803279 187.515803,82.1676066 C182.190164,65.7752131 177.240131,50.5390164 173.42741,38.807082 L130.068984,52.8996721 L144.159475,96.256" fill="#62803A"></path></g></svg>'''
 
     valid_events = [
         events.PullRequestCloseEvent,
         events.PullRequestMergeEvent,
         events.PullRequestUpdateEvent,
         events.PullRequestCommentEvent,
         events.PullRequestReviewEvent,
         events.PullRequestCreateEvent,
         events.RepoPushEvent,
         events.RepoCreateEvent,
     ]
 
     def send_event(self, event):
         log.debug('handling event %s with integration %s', event.name, self)
 
         if event.__class__ not in self.valid_events:
             log.debug('event %r not present in valid event list (%s)', event, self.valid_events)
             return
 
         if not self.event_enabled(event):
             return
 
         data = event.as_dict()
 
         # defaults
         title = '*%s* caused a *%s* event' % (
             data['actor']['username'], event.name)
         text = '*%s* caused a *%s* event' % (
             data['actor']['username'], event.name)
         fields = None
         overrides = None
 
         if isinstance(event, events.PullRequestCommentEvent):
             (title, text, fields, overrides) \
                 = self.format_pull_request_comment_event(event, data)
         elif isinstance(event, events.PullRequestCommentEditEvent):
             (title, text, fields, overrides) \
                 = self.format_pull_request_comment_event(event, data)
         elif isinstance(event, events.PullRequestReviewEvent):
             title, text = self.format_pull_request_review_event(event, data)
         elif isinstance(event, events.PullRequestEvent):
             title, text = self.format_pull_request_event(event, data)
         elif isinstance(event, events.RepoPushEvent):
             title, text = self.format_repo_push_event(data)
         elif isinstance(event, events.RepoCreateEvent):
             title, text = self.format_repo_create_event(data)
         else:
             log.error('unhandled event type: %r', event)
 
         run_task(post_text_to_slack, self.settings, title, text, fields, overrides)
 
     def settings_schema(self):
         schema = SlackSettingsSchema()
         schema.add(colander.SchemaNode(
             colander.Set(),
             widget=CheckboxChoiceWidgetDesc(
                 values=sorted(
                     [(e.name, e.display_name, e.description) for e in self.valid_events]
                 ),
             ),
             description="List of events activated for this integration",
             name='events'
         ))
 
         return schema
 
     def format_pull_request_comment_event(self, event, data):
         comment_text = data['comment']['text']
         if len(comment_text) > 200:
             comment_text = '<{comment_url}|{comment_text}...>'.format(
                 comment_text=comment_text[:200],
                 comment_url=data['comment']['url'],
             )
 
         fields = None
         overrides = None
         status_text = None
 
         if data['comment']['status']:
             status_color = {
                 'approved': '#0ac878',
                 'rejected': '#e85e4d'}.get(data['comment']['status'])
 
             if status_color:
                 overrides = {"color": status_color}
 
             status_text = data['comment']['status']
 
         if data['comment']['file']:
             fields = [
                 {
                     "title": "file",
                     "value": data['comment']['file']
                 },
                 {
                     "title": "line",
                     "value": data['comment']['line']
                 }
             ]
 
         template = Template(textwrap.dedent(r'''
         *${data['actor']['username']}* left ${data['comment']['type']} on pull request <${data['pullrequest']['url']}|#${data['pullrequest']['pull_request_id']}>:
         '''))
         title = render_with_traceback(
             template, data=data, comment=event.comment)
 
         template = Template(textwrap.dedent(r'''
         *pull request title*: ${pr_title}
         % if status_text:
         *submitted status*: `${status_text}`
         % endif
         >>> ${comment_text}
         '''))
         text = render_with_traceback(
             template,
             comment_text=comment_text,
             pr_title=data['pullrequest']['title'],
             status_text=status_text)
 
         return title, text, fields, overrides
 
     def format_pull_request_review_event(self, event, data):
         template = Template(textwrap.dedent(r'''
         *${data['actor']['username']}* changed status of pull request <${data['pullrequest']['url']}|#${data['pullrequest']['pull_request_id']} to `${data['pullrequest']['status']}`>:
         '''))
         title = render_with_traceback(template, data=data)
 
         template = Template(textwrap.dedent(r'''
         *pull request title*: ${pr_title}
         '''))
         text = render_with_traceback(
             template,
             pr_title=data['pullrequest']['title'])
 
         return title, text
 
     def format_pull_request_event(self, event, data):
         action = {
             events.PullRequestCloseEvent: 'closed',
             events.PullRequestMergeEvent: 'merged',
             events.PullRequestUpdateEvent: 'updated',
             events.PullRequestCreateEvent: 'created',
         }.get(event.__class__, str(event.__class__))
 
         template = Template(textwrap.dedent(r'''
         *${data['actor']['username']}* `${action}` pull request <${data['pullrequest']['url']}|#${data['pullrequest']['pull_request_id']}>:
         '''))
         title = render_with_traceback(template, data=data, action=action)
 
         template = Template(textwrap.dedent(r'''
         *pull request title*: ${pr_title}
         %if data['pullrequest']['commits']:
         *commits*: ${len(data['pullrequest']['commits'])}
         %endif
         '''))
         text = render_with_traceback(
             template,
             pr_title=data['pullrequest']['title'],
             data=data)
 
         return title, text
 
     def format_repo_push_event(self, data):
         branches_commits = self.aggregate_branch_data(
             data['push']['branches'], data['push']['commits'])
 
         template = Template(r'''
         *${data['actor']['username']}* pushed to repo <${data['repo']['url']}|${data['repo']['repo_name']}>:
         ''')
         title = render_with_traceback(template, data=data)
 
         text = render_with_traceback(
             REPO_PUSH_TEMPLATE,
             data=data,
             branches_commits=branches_commits,
             html_to_slack_links=html_to_slack_links,
         )
 
         return title, text
 
     def format_repo_create_event(self, data):
         template = Template(r'''
         *${data['actor']['username']}* created new repository ${data['repo']['repo_name']}:
         ''')
         title = render_with_traceback(template, data=data)
 
         template = Template(textwrap.dedent(r'''
         repo_url: ${data['repo']['url']}
         repo_type: ${data['repo']['repo_type']}
         '''))
         text = render_with_traceback(template, data=data)
 
         return title, text
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def post_text_to_slack(settings, title, text, fields=None, overrides=None):
     log.debug('sending %s (%s) to slack %s', title, text, settings['service'])
 
     fields = fields or []
     overrides = overrides or {}
 
     message_data = {
                 "fallback": text,
                 "color": "#427cc9",
                 "pretext": title,
                 #"author_name": "Bobby Tables",
                 #"author_link": "http://flickr.com/bobby/",
                 #"author_icon": "http://flickr.com/icons/bobby.jpg",
                 #"title": "Slack API Documentation",
                 #"title_link": "https://api.slack.com/",
                 "text": text,
                 "fields": fields,
                 #"image_url": "http://my-website.com/path/to/image.jpg",
                 #"thumb_url": "http://example.com/path/to/thumb.png",
                 "footer": "RhodeCode",
                 #"footer_icon": "",
                 "ts": time.time(),
                 "mrkdwn_in": ["pretext", "text"]
     }
     message_data.update(overrides)
     json_message = {
         "icon_emoji": settings.get('icon_emoji', ':studio_microphone:'),
         "channel": settings.get('channel', ''),
         "username": settings.get('username', 'Rhodecode'),
         "attachments": [message_data]
     }
     req_session = requests_retry_call()
     resp = req_session.post(settings['service'], json=json_message, timeout=60)
     resp.raise_for_status()  # raise exception on a failed request

rhodecode/integrations/types/webhook.py

@@ -1,266 +1,266 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-from __future__ import unicode_literals
+
 
 import deform.widget
 import logging
 import colander
 
 import rhodecode
 from rhodecode import events
 from rhodecode.lib.colander_utils import strip_whitespace
 from rhodecode.model.validation_schema.widgets import CheckboxChoiceWidgetDesc
 from rhodecode.translation import _
 from rhodecode.integrations.types.base import (
     IntegrationTypeBase, get_auth, get_web_token, get_url_vars,
     WebhookDataHandler, WEBHOOK_URL_VARS, requests_retry_call)
 from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask
 from rhodecode.model.validation_schema import widgets
 
 log = logging.getLogger(__name__)
 
 
 # updating this required to update the `common_vars` passed in url calling func
 
 URL_VARS = get_url_vars(WEBHOOK_URL_VARS)
 
 
 class WebhookSettingsSchema(colander.Schema):
     url = colander.SchemaNode(
         colander.String(),
         title=_('Webhook URL'),
         description=
             _('URL to which Webhook should submit data. If used some of the '
               'variables would trigger multiple calls, like ${branch} or '
               '${commit_id}. Webhook will be called as many times as unique '
               'objects in data in such cases.'),
         missing=colander.required,
         required=True,
         preparer=strip_whitespace,
         validator=colander.url,
         widget=widgets.CodeMirrorWidget(
             help_block_collapsable_name='Show url variables',
             help_block_collapsable=(
                 'E.g http://my-serv.com/trigger_job/${{event_name}}'
                 '?PR_ID=${{pull_request_id}}'
                 '\nFull list of vars:\n{}'.format(URL_VARS)),
             codemirror_mode='text',
             codemirror_options='{"lineNumbers": false, "lineWrapping": true}'),
     )
     secret_token = colander.SchemaNode(
         colander.String(),
         title=_('Secret Token'),
         description=_('Optional string used to validate received payloads. '
                       'It will be sent together with event data in JSON'),
         default='',
         missing='',
         widget=deform.widget.TextInputWidget(
             placeholder='e.g. secret_token'
         ),
     )
     username = colander.SchemaNode(
         colander.String(),
         title=_('Username'),
         description=_('Optional username to authenticate the call.'),
         default='',
         missing='',
         widget=deform.widget.TextInputWidget(
             placeholder='e.g. admin'
         ),
     )
     password = colander.SchemaNode(
         colander.String(),
         title=_('Password'),
         description=_('Optional password to authenticate the call.'),
         default='',
         missing='',
         widget=deform.widget.PasswordWidget(
             placeholder='e.g. secret.',
             redisplay=True,
         ),
     )
     custom_header_key = colander.SchemaNode(
         colander.String(),
         title=_('Custom Header Key'),
         description=_('Custom Header name to be set when calling endpoint.'),
         default='',
         missing='',
         widget=deform.widget.TextInputWidget(
             placeholder='e.g: Authorization'
         ),
     )
     custom_header_val = colander.SchemaNode(
         colander.String(),
         title=_('Custom Header Value'),
         description=_('Custom Header value to be set when calling endpoint.'),
         default='',
         missing='',
         widget=deform.widget.TextInputWidget(
             placeholder='e.g. Basic XxXxXx'
         ),
     )
     method_type = colander.SchemaNode(
         colander.String(),
         title=_('Call Method'),
         description=_('Select a HTTP method to use when calling the Webhook.'),
         default='post',
         missing='',
         widget=deform.widget.RadioChoiceWidget(
             values=[('get', 'GET'), ('post', 'POST'), ('put', 'PUT')],
             inline=True
         ),
     )
 
 
 class WebhookIntegrationType(IntegrationTypeBase):
     key = 'webhook'
     display_name = _('Webhook')
     description = _('send JSON data to a url endpoint')
 
     @classmethod
     def icon(cls):
         return '''<?xml version="1.0" encoding="UTF-8" standalone="no"?><svg viewBox="0 0 256 239" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid"><g><path d="M119.540432,100.502743 C108.930124,118.338815 98.7646301,135.611455 88.3876025,152.753617 C85.7226696,157.154315 84.4040417,160.738531 86.5332204,166.333309 C92.4107024,181.787152 84.1193605,196.825836 68.5350381,200.908244 C53.8383677,204.759349 39.5192953,195.099955 36.6032893,179.365384 C34.0194114,165.437749 44.8274148,151.78491 60.1824106,149.608284 C61.4694072,149.424428 62.7821041,149.402681 64.944891,149.240571 C72.469175,136.623655 80.1773157,123.700312 88.3025935,110.073173 C73.611854,95.4654658 64.8677898,78.3885437 66.803227,57.2292132 C68.1712787,42.2715849 74.0527146,29.3462646 84.8033863,18.7517722 C105.393354,-1.53572199 136.805164,-4.82141828 161.048542,10.7510424 C184.333097,25.7086706 194.996783,54.8450075 185.906752,79.7822957 C179.052655,77.9239597 172.151111,76.049808 164.563565,73.9917997 C167.418285,60.1274266 165.306899,47.6765751 155.95591,37.0109123 C149.777932,29.9690049 141.850349,26.2780332 132.835442,24.9178894 C114.764113,22.1877169 97.0209573,33.7983633 91.7563309,51.5355878 C85.7800012,71.6669027 94.8245623,88.1111998 119.540432,100.502743 L119.540432,100.502743 Z" fill="#C73A63"></path><path d="M149.841194,79.4106285 C157.316054,92.5969067 164.905578,105.982857 172.427885,119.246236 C210.44865,107.483365 239.114472,128.530009 249.398582,151.063322 C261.81978,178.282014 253.328765,210.520191 228.933162,227.312431 C203.893073,244.551464 172.226236,241.605803 150.040866,219.46195 C155.694953,214.729124 161.376716,209.974552 167.44794,204.895759 C189.360489,219.088306 208.525074,218.420096 222.753207,201.614016 C234.885769,187.277151 234.622834,165.900356 222.138374,151.863988 C207.730339,135.66681 188.431321,135.172572 165.103273,150.721309 C155.426087,133.553447 145.58086,116.521995 136.210101,99.2295848 C133.05093,93.4015266 129.561608,90.0209366 122.440622,88.7873178 C110.547271,86.7253555 102.868785,76.5124151 102.408155,65.0698097 C101.955433,53.7537294 108.621719,43.5249733 119.04224,39.5394355 C129.363912,35.5914599 141.476705,38.7783085 148.419765,47.554004 C154.093621,54.7244134 155.896602,62.7943365 152.911402,71.6372484 C152.081082,74.1025091 151.00562,76.4886916 149.841194,79.4106285 L149.841194,79.4106285 Z" fill="#4B4B4B"></path><path d="M167.706921,187.209935 L121.936499,187.209935 C117.54964,205.253587 108.074103,219.821756 91.7464461,229.085759 C79.0544063,236.285822 65.3738898,238.72736 50.8136292,236.376762 C24.0061432,232.053165 2.08568567,207.920497 0.156179306,180.745298 C-2.02835403,149.962159 19.1309765,122.599149 47.3341915,116.452801 C49.2814904,123.524363 51.2485589,130.663141 53.1958579,137.716911 C27.3195169,150.919004 18.3639187,167.553089 25.6054984,188.352614 C31.9811726,206.657224 50.0900643,216.690262 69.7528413,212.809503 C89.8327554,208.847688 99.9567329,192.160226 98.7211371,165.37844 C117.75722,165.37844 136.809118,165.180745 155.847178,165.475311 C163.280522,165.591951 169.019617,164.820939 174.620326,158.267339 C183.840836,147.48306 200.811003,148.455721 210.741239,158.640984 C220.88894,169.049642 220.402609,185.79839 209.663799,195.768166 C199.302587,205.38802 182.933414,204.874012 173.240413,194.508846 C171.247644,192.37176 169.677943,189.835329 167.706921,187.209935 L167.706921,187.209935 Z" fill="#4A4A4A"></path></g></svg>'''
 
     valid_events = [
         events.PullRequestCloseEvent,
         events.PullRequestMergeEvent,
         events.PullRequestUpdateEvent,
         events.PullRequestCommentEvent,
         events.PullRequestCommentEditEvent,
         events.PullRequestReviewEvent,
         events.PullRequestCreateEvent,
         events.RepoPushEvent,
         events.RepoCreateEvent,
         events.RepoCommitCommentEvent,
         events.RepoCommitCommentEditEvent,
     ]
 
     def settings_schema(self):
         schema = WebhookSettingsSchema()
         schema.add(colander.SchemaNode(
             colander.Set(),
             widget=CheckboxChoiceWidgetDesc(
                 values=sorted(
                     [(e.name, e.display_name, e.description) for e in self.valid_events]
                 ),
             ),
             description="List of events activated for this integration",
             name='events'
         ))
         return schema
 
     def send_event(self, event):
         log.debug('handling event %s with integration %s', event.name, self)
 
         if event.__class__ not in self.valid_events:
             log.debug('event %r not present in valid event list (%s)', event, self.valid_events)
             return
 
         if not self.event_enabled(event):
             return
 
         data = event.as_dict()
         template_url = self.settings['url']
 
         headers = {}
         head_key = self.settings.get('custom_header_key')
         head_val = self.settings.get('custom_header_val')
         if head_key and head_val:
             headers = {head_key: head_val}
 
         handler = WebhookDataHandler(template_url, headers)
 
         url_calls = handler(event, data)
         log.debug('Webhook: calling following urls: %s', [x[0] for x in url_calls])
 
         run_task(post_to_webhook, url_calls, self.settings)
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def post_to_webhook(url_calls, settings):
     """
     Example data::
 
         {'actor': {'user_id': 2, 'username': u'admin'},
          'actor_ip': u'192.168.157.1',
          'name': 'repo-push',
          'push': {'branches': [{'name': u'default',
                                 'url': 'http://rc.local:8080/hg-repo/changelog?branch=default'}],
                   'commits': [{'author': u'Marcin Kuzminski <[e-mail]>',
                                'branch': u'default',
                                'date': datetime.datetime(2017, 11, 30, 12, 59, 48),
                                'issues': [],
                                'mentions': [],
                                'message': u'commit Thu 30 Nov 2017 13:59:48 CET',
                                'message_html': u'commit Thu 30 Nov 2017 13:59:48 CET',
                                'message_html_title': u'commit Thu 30 Nov 2017 13:59:48 CET',
                                'parents': [{'raw_id': '431b772a5353dad9974b810dd3707d79e3a7f6e0'}],
                                'permalink_url': u'http://rc.local:8080/_7/changeset/a815cc738b9651eb5ffbcfb1ce6ccd7c701a5ddf',
                                'raw_id': 'a815cc738b9651eb5ffbcfb1ce6ccd7c701a5ddf',
                                'refs': {'bookmarks': [],
                                         'branches': [u'default'],
                                         'tags': [u'tip']},
                                'reviewers': [],
                                'revision': 9L,
                                'short_id': 'a815cc738b96',
                                'url': u'http://rc.local:8080/hg-repo/changeset/a815cc738b9651eb5ffbcfb1ce6ccd7c701a5ddf'}],
                   'issues': {}},
          'repo': {'extra_fields': '',
                   'permalink_url': u'http://rc.local:8080/_7',
                   'repo_id': 7,
                   'repo_name': u'hg-repo',
                   'repo_type': u'hg',
                   'url': u'http://rc.local:8080/hg-repo'},
          'server_url': u'http://rc.local:8080',
          'utc_timestamp': datetime.datetime(2017, 11, 30, 13, 0, 1, 569276)
          }
     """
 
     call_headers = {
         'User-Agent': 'RhodeCode-webhook-caller/{}'.format(rhodecode.__version__)
     }  # updated below with custom ones, allows override
 
     auth = get_auth(settings)
     token = get_web_token(settings)
 
     for url, headers, data in url_calls:
         req_session = requests_retry_call()
 
         method = settings.get('method_type') or 'post'
         call_method = getattr(req_session, method)
 
         headers = headers or {}
         call_headers.update(headers)
 
         log.debug('calling Webhook with method: %s, and auth:%s', call_method, auth)
         if settings.get('log_data'):
             log.debug('calling webhook with data: %s', data)
         resp = call_method(url, json={
             'token': token,
             'event': data
         }, headers=call_headers, auth=auth, timeout=60)
         log.debug('Got Webhook response: %s', resp)
 
         try:
             resp.raise_for_status()  # raise exception on a failed request
         except Exception:
             log.error(resp.text)
             raise

rhodecode/lib/_vendor/authomatic/extras/flask.py

@@ -1,47 +1,47 b''
 # -*- coding: utf-8 -*-
 """
 |flask| Extras
 --------------
 
 Utilities you can use when using this library with the |flask|_ framework.
 
 Thanks to `Mark Steve Samson <http://marksteve.com>`_.
 """
 
-from __future__ import absolute_import
+
 from functools import wraps
 
 from authomatic.adapters import WerkzeugAdapter
 from authomatic import Authomatic
 from flask import make_response, request, session
 
 
 class FlaskAuthomatic(Authomatic):
     """
     Flask Plugin for authomatic support.
     """
 
     result = None
 
     def login(self, *login_args, **login_kwargs):
         """
         Decorator for Flask view functions.
         """
 
         def decorator(f):
             @wraps(f)
             def decorated(*args, **kwargs):
                 self.response = make_response()
                 adapter = WerkzeugAdapter(request, self.response)
                 login_kwargs.setdefault('session', session)
                 login_kwargs.setdefault('session_saver', self.session_saver)
                 self.result = super(FlaskAuthomatic, self).login(
                     adapter,
                     *login_args,
                     **login_kwargs)
                 return f(*args, **kwargs)
             return decorated
         return decorator
 
     def session_saver(self):
         session.modified = True

rhodecode/lib/_vendor/authomatic/extras/gae/openid.py

@@ -1,156 +1,156 b''
 # -*- coding: utf-8 -*-
 
 # We need absolute import to import from openid library which has the same
 # name as this module
-from __future__ import absolute_import
+
 import logging
 import datetime
 
 from google.appengine.ext import ndb
 import openid.store.interface
 
 
 class NDBOpenIDStore(ndb.Expando, openid.store.interface.OpenIDStore):
     """
     |gae| `NDB <https://developers.google.com/appengine/docs/python/ndb/>`_
     based implementation of the :class:`openid.store.interface.OpenIDStore`
     interface of the `python-openid`_ library.
     """
 
     serialized = ndb.StringProperty()
     expiration_date = ndb.DateTimeProperty()
     # we need issued to sort by most recently issued
     issued = ndb.IntegerProperty()
 
     @staticmethod
     def _log(*args, **kwargs):
         pass
 
     @classmethod
     def storeAssociation(cls, server_url, association):
         # store an entity with key = server_url
 
         issued = datetime.datetime.fromtimestamp(association.issued)
         lifetime = datetime.timedelta(0, association.lifetime)
 
         expiration_date = issued + lifetime
         entity = cls.get_or_insert(
             association.handle, parent=ndb.Key(
                 'ServerUrl', server_url))
 
         entity.serialized = association.serialize()
         entity.expiration_date = expiration_date
         entity.issued = association.issued
 
         cls._log(
             logging.DEBUG,
             u'NDBOpenIDStore: Putting OpenID association to datastore.')
 
         entity.put()
 
     @classmethod
     def cleanupAssociations(cls):
         # query for all expired
         cls._log(
             logging.DEBUG,
             u'NDBOpenIDStore: Querying datastore for OpenID associations.')
         query = cls.query(cls.expiration_date <= datetime.datetime.now())
 
         # fetch keys only
         expired = query.fetch(keys_only=True)
 
         # delete all expired
         cls._log(
             logging.DEBUG,
             u'NDBOpenIDStore: Deleting expired OpenID associations from datastore.')
         ndb.delete_multi(expired)
 
         return len(expired)
 
     @classmethod
     def getAssociation(cls, server_url, handle=None):
         cls.cleanupAssociations()
 
         if handle:
             key = ndb.Key('ServerUrl', server_url, cls, handle)
             cls._log(
                 logging.DEBUG,
                 u'NDBOpenIDStore: Getting OpenID association from datastore by key.')
             entity = key.get()
         else:
             # return most recently issued association
             cls._log(
                 logging.DEBUG,
                 u'NDBOpenIDStore: Querying datastore for OpenID associations by ancestor.')
             entity = cls.query(ancestor=ndb.Key(
                 'ServerUrl', server_url)).order(-cls.issued).get()
 
         if entity and entity.serialized:
             return openid.association.Association.deserialize(
                 entity.serialized)
 
     @classmethod
     def removeAssociation(cls, server_url, handle):
         key = ndb.Key('ServerUrl', server_url, cls, handle)
         cls._log(
             logging.DEBUG,
             u'NDBOpenIDStore: Getting OpenID association from datastore by key.')
         if key.get():
             cls._log(
                 logging.DEBUG,
                 u'NDBOpenIDStore: Deleting OpenID association from datastore.')
             key.delete()
             return True
 
     @classmethod
     def useNonce(cls, server_url, timestamp, salt):
 
         # check whether there is already an entity with the same ancestor path
         # in the datastore
         key = ndb.Key(
             'ServerUrl',
             str(server_url) or 'x',
             'TimeStamp',
             str(timestamp),
             cls,
             str(salt))
 
         cls._log(
             logging.DEBUG,
             u'NDBOpenIDStore: Getting OpenID nonce from datastore by key.')
         result = key.get()
 
         if result:
             # if so, the nonce is not valid so return False
             cls._log(
                 logging.WARNING,
                 u'NDBOpenIDStore: Nonce was already used!')
             return False
         else:
             # if not, store the key to datastore and return True
             nonce = cls(key=key)
             nonce.expiration_date = datetime.datetime.fromtimestamp(
                 timestamp) + datetime.timedelta(0, openid.store.nonce.SKEW)
             cls._log(
                 logging.DEBUG,
                 u'NDBOpenIDStore: Putting new nonce to datastore.')
             nonce.put()
             return True
 
     @classmethod
     def cleanupNonces(cls):
         # get all expired nonces
         cls._log(
             logging.DEBUG,
             u'NDBOpenIDStore: Querying datastore for OpenID nonces ordered by expiration date.')
         expired = cls.query().filter(
             cls.expiration_date <= datetime.datetime.now()).fetch(
             keys_only=True)
 
         # delete all expired
         cls._log(
             logging.DEBUG,
             u'NDBOpenIDStore: Deleting expired OpenID nonces from datastore.')
         ndb.delete_multi(expired)
 
         return len(expired)

rhodecode/lib/_vendor/authomatic/providers/openid.py

@@ -1,505 +1,505 b''
 # -*- coding: utf-8 -*-
 """
 |openid| Providers
 ----------------------------------
 
 Providers which implement the |openid|_ protocol based on the
 `python-openid`_ library.
 
 .. warning::
 
     This providers are dependent on the |pyopenid|_ package.
 
 .. autosummary::
 
     OpenID
     Yahoo
     Google
 
 """
 
 # We need absolute import to import from openid library which has the same
 # name as this module
-from __future__ import absolute_import
+
 import datetime
 import logging
 import time
 
 from openid import oidutil
 from openid.consumer import consumer
 from openid.extensions import ax, pape, sreg
 from openid.association import Association
 
 from authomatic import providers
 from authomatic.exceptions import FailureError, CancellationError, OpenIDError
 
 
 __all__ = ['OpenID', 'Yahoo', 'Google']
 
 
 # Suppress openid logging.
 oidutil.log = lambda message, level=0: None
 
 
 REALM_HTML = \
     """
 <!DOCTYPE html>
 <html>
     <head>
         <meta http-equiv="X-XRDS-Location" content="{xrds_location}" />
     </head>
     <body>{body}</body>
 </html>
 """
 
 
 XRDS_XML = \
     """
 <?xml version="1.0" encoding="UTF-8"?>
 <xrds:XRDS
     xmlns:xrds="xri://$xrds"
     xmlns:openid="http://openid.net/xmlns/1.0"
     xmlns="xri://$xrd*($v*2.0)">
     <XRD>
         <Service priority="1">
             <Type>http://specs.openid.net/auth/2.0/return_to</Type>
             <URI>{return_to}</URI>
         </Service>
     </XRD>
 </xrds:XRDS>
 """
 
 
 class SessionOpenIDStore(object):
     """
     A very primitive session-based implementation of the.
 
     :class:`openid.store.interface.OpenIDStore` interface of the
     `python-openid`_ library.
 
     .. warning::
 
         Nonces get verified only by their timeout. Use on your own risk!
 
     """
 
     @staticmethod
     def _log(level, message):
         return None
 
     ASSOCIATION_KEY = ('authomatic.providers.openid.SessionOpenIDStore:'
                        'association')
 
     def __init__(self, session, nonce_timeout=None):
         """
         :param int nonce_timeout:
 
             Nonces older than this in seconds will be considered expired.
             Default is 600.
         """
         self.session = session
         self.nonce_timeout = nonce_timeout or 600
 
     def storeAssociation(self, server_url, association):
         self._log(logging.DEBUG,
                   'SessionOpenIDStore: Storing association to session.')
 
         serialized = association.serialize()
         decoded = serialized.decode('latin-1')
 
         assoc = decoded
         # assoc = serialized
 
         # Always store only one association as a tuple.
         self.session[self.ASSOCIATION_KEY] = (server_url, association.handle,
                                               assoc)
 
     def getAssociation(self, server_url, handle=None):
         # Try to get association.
         assoc = self.session.get(self.ASSOCIATION_KEY)
         if assoc and assoc[0] == server_url:
             # If found deserialize and return it.
             self._log(logging.DEBUG, u'SessionOpenIDStore: Association found.')
             return Association.deserialize(assoc[2].encode('latin-1'))
         else:
             self._log(logging.DEBUG,
                       u'SessionOpenIDStore: Association not found.')
 
     def removeAssociation(self, server_url, handle):
         # Just inform the caller that it's gone.
         return True
 
     def useNonce(self, server_url, timestamp, salt):
         # Evaluate expired nonces as false.
         age = int(time.time()) - int(timestamp)
         if age < self.nonce_timeout:
             return True
         else:
             self._log(logging.ERROR, u'SessionOpenIDStore: Expired nonce!')
             return False
 
 
 class OpenID(providers.AuthenticationProvider):
     """
     |openid|_ provider based on the `python-openid`_ library.
     """
 
     AX = ['http://axschema.org/contact/email',
           'http://schema.openid.net/contact/email',
           'http://axschema.org/namePerson',
           'http://openid.net/schema/namePerson/first',
           'http://openid.net/schema/namePerson/last',
           'http://openid.net/schema/gender',
           'http://openid.net/schema/language/pref',
           'http://openid.net/schema/contact/web/default',
           'http://openid.net/schema/media/image',
           'http://openid.net/schema/timezone']
 
     AX_REQUIRED = ['http://schema.openid.net/contact/email']
 
     SREG = ['nickname',
             'email',
             'fullname',
             'dob',
             'gender',
             'postcode',
             'country',
             'language',
             'timezone']
 
     PAPE = [
         'http://schemas.openid.net/pape/policies/2007/06/'
         'multi-factor-physical',
         'http://schemas.openid.net/pape/policies/2007/06/multi-factor',
         'http://schemas.openid.net/pape/policies/2007/06/phishing-resistant'
     ]
 
     def __init__(self, *args, **kwargs):
         """
         Accepts additional keyword arguments:
 
         :param store:
             Any object which implements
             :class:`openid.store.interface.OpenIDStore`
             of the `python-openid`_ library.
 
         :param bool use_realm:
             Whether to use `OpenID realm
             <http://openid.net/specs/openid-authentication-2_0-12.html#realms>`_
             If ``True`` the realm HTML document will be accessible at
             ``{current url}?{realm_param}={realm_param}``
             e.g. ``http://example.com/path?realm=realm``.
 
         :param str realm_body:
             Contents of the HTML body tag of the realm.
 
         :param str realm_param:
             Name of the query parameter to be used to serve the realm.
 
         :param str xrds_param:
             The name of the query parameter to be used to serve the
             `XRDS document
             <http://openid.net/specs/openid-authentication-2_0-12.html#XRDS_Sample>`_.
 
         :param list sreg:
             List of strings of optional
             `SREG
             <http://openid.net/specs/openid-simple-registration-extension-1_0.html>`_
             fields.
             Default = :attr:`OpenID.SREG`.
 
         :param list sreg_required:
             List of strings of required
             `SREG
             <http://openid.net/specs/openid-simple-registration-extension-1_0.html>`_
             fields.
             Default = ``[]``.
 
         :param list ax:
             List of strings of optional
             `AX
             <http://openid.net/specs/openid-attribute-exchange-1_0.html>`_
             schemas.
             Default = :attr:`OpenID.AX`.
 
         :param list ax_required:
             List of strings of required
             `AX
             <http://openid.net/specs/openid-attribute-exchange-1_0.html>`_
             schemas.
             Default = :attr:`OpenID.AX_REQUIRED`.
 
         :param list pape:
             of requested
             `PAPE
             <http://openid.net/specs/openid-provider-authentication-policy-extension-1_0.html>`_
             policies.
             Default = :attr:`OpenID.PAPE`.
 
         As well as those inherited from :class:`.AuthenticationProvider`
         constructor.
 
         """
 
         super(OpenID, self).__init__(*args, **kwargs)
 
         # Allow for other openid store implementations.
         self.store = self._kwarg(
             kwargs, 'store', SessionOpenIDStore(
                 self.session))
 
         # Realm
         self.use_realm = self._kwarg(kwargs, 'use_realm', True)
         self.realm_body = self._kwarg(kwargs, 'realm_body', '')
         self.realm_param = self._kwarg(kwargs, 'realm_param', 'realm')
         self.xrds_param = self._kwarg(kwargs, 'xrds_param', 'xrds')
 
         # SREG
         self.sreg = self._kwarg(kwargs, 'sreg', self.SREG)
         self.sreg_required = self._kwarg(kwargs, 'sreg_required', [])
 
         # AX
         self.ax = self._kwarg(kwargs, 'ax', self.AX)
         self.ax_required = self._kwarg(kwargs, 'ax_required', self.AX_REQUIRED)
         # add required schemas to schemas if not already there
         for i in self.ax_required:
             if i not in self.ax:
                 self.ax.append(i)
 
         # PAPE
         self.pape = self._kwarg(kwargs, 'pape', self.PAPE)
 
     @staticmethod
     def _x_user_parser(user, data):
 
         user.first_name = data.get('ax', {}).get(
             'http://openid.net/schema/namePerson/first')
         user.last_name = data.get('ax', {}).get(
             'http://openid.net/schema/namePerson/last')
         user.id = data.get('guid')
         user.link = data.get('ax', {}).get(
             'http://openid.net/schema/contact/web/default')
         user.picture = data.get('ax', {}).get(
             'http://openid.net/schema/media/image')
         user.nickname = data.get('sreg', {}).get('nickname')
         user.country = data.get('sreg', {}).get('country')
         user.postal_code = data.get('sreg', {}).get('postcode')
 
         user.name = data.get('sreg', {}).get('fullname') or \
             data.get('ax', {}).get('http://axschema.org/namePerson')
 
         user.gender = data.get('sreg', {}).get('gender') or \
             data.get('ax', {}).get('http://openid.net/schema/gender')
 
         user.locale = data.get('sreg', {}).get('language') or \
             data.get('ax', {}).get('http://openid.net/schema/language/pref')
 
         user.timezone = data.get('sreg', {}).get('timezone') or \
             data.get('ax', {}).get('http://openid.net/schema/timezone')
 
         user.email = data.get('sreg', {}).get('email') or \
             data.get('ax', {}).get('http://axschema.org/contact/email') or \
             data.get('ax', {}).get('http://schema.openid.net/contact/email')
 
         if data.get('sreg', {}).get('dob'):
             user.birth_date = datetime.datetime.strptime(
                 data.get('sreg', {}).get('dob'),
                 '%Y-%m-%d'
             )
         else:
             user.birth_date = None
 
         return user
 
     @providers.login_decorator
     def login(self):
         # Instantiate consumer
         self.store._log = self._log
         oi_consumer = consumer.Consumer(self.session, self.store)
 
         # handle realm and XRDS if there is only one query parameter
         if self.use_realm and len(self.params) == 1:
             realm_request = self.params.get(self.realm_param)
             xrds_request = self.params.get(self.xrds_param)
         else:
             realm_request = None
             xrds_request = None
 
         # determine type of request
         if realm_request:
             # =================================================================
             # Realm HTML
             # =================================================================
 
             self._log(
                 logging.INFO,
                 u'Writing OpenID realm HTML to the response.')
             xrds_location = '{u}?{x}={x}'.format(u=self.url, x=self.xrds_param)
             self.write(
                 REALM_HTML.format(
                     xrds_location=xrds_location,
                     body=self.realm_body))
 
         elif xrds_request:
             # =================================================================
             # XRDS XML
             # =================================================================
 
             self._log(
                 logging.INFO,
                 u'Writing XRDS XML document to the response.')
             self.set_header('Content-Type', 'application/xrds+xml')
             self.write(XRDS_XML.format(return_to=self.url))
 
         elif self.params.get('openid.mode'):
             # =================================================================
             # Phase 2 after redirect
             # =================================================================
 
             self._log(
                 logging.INFO,
                 u'Continuing OpenID authentication procedure after redirect.')
 
             # complete the authentication process
             response = oi_consumer.complete(self.params, self.url)
 
             # on success
             if response.status == consumer.SUCCESS:
 
                 data = {}
 
                 # get user ID
                 data['guid'] = response.getDisplayIdentifier()
 
                 self._log(logging.INFO, u'Authentication successful.')
 
                 # get user data from AX response
                 ax_response = ax.FetchResponse.fromSuccessResponse(response)
                 if ax_response and ax_response.data:
                     self._log(logging.INFO, u'Got AX data.')
                     ax_data = {}
                     # convert iterable values to their first item
                     for k, v in ax_response.data.items():
                         if v and isinstance(v, (list, tuple)):
                             ax_data[k] = v[0]
                     data['ax'] = ax_data
 
                 # get user data from SREG response
                 sreg_response = sreg.SRegResponse.fromSuccessResponse(response)
                 if sreg_response and sreg_response.data:
                     self._log(logging.INFO, u'Got SREG data.')
                     data['sreg'] = sreg_response.data
 
                 # get data from PAPE response
                 pape_response = pape.Response.fromSuccessResponse(response)
                 if pape_response and pape_response.auth_policies:
                     self._log(logging.INFO, u'Got PAPE data.')
                     data['pape'] = pape_response.auth_policies
 
                 # create user
                 self._update_or_create_user(data)
 
                 # =============================================================
                 # We're done!
                 # =============================================================
 
             elif response.status == consumer.CANCEL:
                 raise CancellationError(
                     u'User cancelled the verification of ID "{0}"!'.format(
                         response.getDisplayIdentifier()))
 
             elif response.status == consumer.FAILURE:
                 raise FailureError(response.message)
 
         elif self.identifier:  # As set in AuthenticationProvider.__init__
             # =================================================================
             # Phase 1 before redirect
             # =================================================================
 
             self._log(
                 logging.INFO,
                 u'Starting OpenID authentication procedure.')
 
             # get AuthRequest object
             try:
                 auth_request = oi_consumer.begin(self.identifier)
             except consumer.DiscoveryFailure as e:
                 raise FailureError(
                     u'Discovery failed for identifier {0}!'.format(
                         self.identifier
                     ),
                     url=self.identifier,
                     original_message=e.message)
 
             self._log(
                 logging.INFO,
                 u'Service discovery for identifier {0} successful.'.format(
                     self.identifier))
 
             # add SREG extension
             # we need to remove required fields from optional fields because
             # addExtension then raises an error
             self.sreg = [i for i in self.sreg if i not in self.sreg_required]
             auth_request.addExtension(
                 sreg.SRegRequest(
                     optional=self.sreg,
                     required=self.sreg_required)
             )
 
             # add AX extension
             ax_request = ax.FetchRequest()
             # set AX schemas
             for i in self.ax:
                 required = i in self.ax_required
                 ax_request.add(ax.AttrInfo(i, required=required))
             auth_request.addExtension(ax_request)
 
             # add PAPE extension
             auth_request.addExtension(pape.Request(self.pape))
 
             # prepare realm and return_to URLs
             if self.use_realm:
                 realm = return_to = '{u}?{r}={r}'.format(
                     u=self.url, r=self.realm_param)
             else:
                 realm = return_to = self.url
 
             url = auth_request.redirectURL(realm, return_to)
 
             if auth_request.shouldSendRedirect():
                 # can be redirected
                 url = auth_request.redirectURL(realm, return_to)
                 self._log(
                     logging.INFO,
                     u'Redirecting user to {0}.'.format(url))
                 self.redirect(url)
             else:
                 # must be sent as POST
                 # this writes a html post form with auto-submit
                 self._log(
                     logging.INFO,
                     u'Writing an auto-submit HTML form to the response.')
                 form = auth_request.htmlMarkup(
                     realm, return_to, False, dict(
                         id='openid_form'))
                 self.write(form)
         else:
             raise OpenIDError('No identifier specified!')
 
 
 class Yahoo(OpenID):
     """
     Yahoo  :class:`.OpenID` provider with the :attr:`.identifier` predefined to
     ``"me.yahoo.com"``.
     """
 
     identifier = 'me.yahoo.com'
 
 
 class Google(OpenID):
     """
     Google :class:`.OpenID` provider with the :attr:`.identifier` predefined to
     ``"https://www.google.com/accounts/o8/id"``.
     """
 
     identifier = 'https://www.google.com/accounts/o8/id'

rhodecode/lib/_vendor/authomatic/six.py

@@ -1,839 +1,839 b''
 # -*- coding: utf-8 -*-
 """Utilities for writing code that runs on Python 2 and 3"""
 
 # Copyright (c) 2010-2015 Benjamin Peterson
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in all
 # copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-from __future__ import absolute_import
+
 
 import functools
 import itertools
 import operator
 import sys
 import types
 
 __author__ = "Benjamin Peterson <[e-mail]>"
 __version__ = "1.9.0"
 
 
 # Useful for very coarse version differentiation.
 PY2 = sys.version_info[0] == 2
 PY3 = sys.version_info[0] == 3
 
 if PY3:
     string_types = str,
     integer_types = int,
     class_types = type,
     text_type = str
     binary_type = bytes
 
     MAXSIZE = sys.maxsize
 else:
     string_types = basestring,
     integer_types = (int, long)
     class_types = (type, types.ClassType)
     text_type = unicode
     binary_type = str
 
     if sys.platform.startswith("java"):
         # Jython always uses 32 bits.
         MAXSIZE = int((1 << 31) - 1)
     else:
         # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
         class X(object):
             def __len__(self):
                 return 1 << 31
         try:
             len(X())
         except OverflowError:
             # 32-bit
             MAXSIZE = int((1 << 31) - 1)
         else:
             # 64-bit
             MAXSIZE = int((1 << 63) - 1)
         del X
 
 
 def _add_doc(func, doc):
     """Add documentation to a function."""
     func.__doc__ = doc
 
 
 def _import_module(name):
     """Import module, returning the module after the last dot."""
     __import__(name)
     return sys.modules[name]
 
 
 class _LazyDescr(object):
 
     def __init__(self, name):
         self.name = name
 
     def __get__(self, obj, tp):
         result = self._resolve()
         setattr(obj, self.name, result) # Invokes __set__.
         try:
             # This is a bit ugly, but it avoids running this again by
             # removing this descriptor.
             delattr(obj.__class__, self.name)
         except AttributeError:
             pass
         return result
 
 
 class MovedModule(_LazyDescr):
 
     def __init__(self, name, old, new=None):
         super(MovedModule, self).__init__(name)
         if PY3:
             if new is None:
                 new = name
             self.mod = new
         else:
             self.mod = old
 
     def _resolve(self):
         return _import_module(self.mod)
 
     def __getattr__(self, attr):
         _module = self._resolve()
         value = getattr(_module, attr)
         setattr(self, attr, value)
         return value
 
 
 class _LazyModule(types.ModuleType):
 
     def __init__(self, name):
         super(_LazyModule, self).__init__(name)
         self.__doc__ = self.__class__.__doc__
 
     def __dir__(self):
         attrs = ["__doc__", "__name__"]
         attrs += [attr.name for attr in self._moved_attributes]
         return attrs
 
     # Subclasses should override this
     _moved_attributes = []
 
 
 class MovedAttribute(_LazyDescr):
 
     def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
         super(MovedAttribute, self).__init__(name)
         if PY3:
             if new_mod is None:
                 new_mod = name
             self.mod = new_mod
             if new_attr is None:
                 if old_attr is None:
                     new_attr = name
                 else:
                     new_attr = old_attr
             self.attr = new_attr
         else:
             self.mod = old_mod
             if old_attr is None:
                 old_attr = name
             self.attr = old_attr
 
     def _resolve(self):
         module = _import_module(self.mod)
         return getattr(module, self.attr)
 
 
 class _SixMetaPathImporter(object):
     """
     A meta path importer to import six.moves and its submodules.
 
     This class implements a PEP302 finder and loader. It should be compatible
     with Python 2.5 and all existing versions of Python3
     """
     def __init__(self, six_module_name):
         self.name = six_module_name
         self.known_modules = {}
 
     def _add_module(self, mod, *fullnames):
         for fullname in fullnames:
             self.known_modules[self.name + "." + fullname] = mod
 
     def _get_module(self, fullname):
         return self.known_modules[self.name + "." + fullname]
 
     def find_module(self, fullname, path=None):
         if fullname in self.known_modules:
             return self
         return None
 
     def __get_module(self, fullname):
         try:
             return self.known_modules[fullname]
         except KeyError:
             raise ImportError("This loader does not know module " + fullname)
 
     def load_module(self, fullname):
         try:
             # in case of a reload
             return sys.modules[fullname]
         except KeyError:
             pass
         mod = self.__get_module(fullname)
         if isinstance(mod, MovedModule):
             mod = mod._resolve()
         else:
             mod.__loader__ = self
         sys.modules[fullname] = mod
         return mod
 
     def is_package(self, fullname):
         """
         Return true, if the named module is a package.
 
         We need this method to get correct spec objects with
         Python 3.4 (see PEP451)
         """
         return hasattr(self.__get_module(fullname), "__path__")
 
     def get_code(self, fullname):
         """Return None
 
         Required, if is_package is implemented"""
         self.__get_module(fullname)  # eventually raises ImportError
         return None
     get_source = get_code  # same as get_code
 
 _importer = _SixMetaPathImporter(__name__)
 
 
 class _MovedItems(_LazyModule):
     """Lazy loading of moved objects"""
     __path__ = []  # mark as package
 
 
 _moved_attributes = [
     MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
     MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
     MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
     MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
     MovedAttribute("intern", "__builtin__", "sys"),
     MovedAttribute("map", "itertools", "builtins", "imap", "map"),
     MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
     MovedAttribute("reload_module", "__builtin__", "imp", "reload"),
     MovedAttribute("reduce", "__builtin__", "functools"),
     MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
     MovedAttribute("StringIO", "StringIO", "io"),
     MovedAttribute("UserDict", "UserDict", "collections"),
     MovedAttribute("UserList", "UserList", "collections"),
     MovedAttribute("UserString", "UserString", "collections"),
     MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
     MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
     MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),
 
     MovedModule("builtins", "__builtin__"),
     MovedModule("configparser", "ConfigParser"),
     MovedModule("copyreg", "copy_reg"),
     MovedModule("dbm_gnu", "gdbm", "dbm.gnu"),
     MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"),
     MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
     MovedModule("http_cookies", "Cookie", "http.cookies"),
     MovedModule("html_entities", "htmlentitydefs", "html.entities"),
     MovedModule("html_parser", "HTMLParser", "html.parser"),
     MovedModule("http_client", "httplib", "http.client"),
     MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
     MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"),
     MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
     MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
     MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
     MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
     MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
     MovedModule("cPickle", "cPickle", "pickle"),
     MovedModule("queue", "Queue"),
     MovedModule("reprlib", "repr"),
     MovedModule("socketserver", "SocketServer"),
     MovedModule("_thread", "thread", "_thread"),
     MovedModule("tkinter", "Tkinter"),
     MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
     MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
     MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
     MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
     MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
     MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"),
     MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
     MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
     MovedModule("tkinter_colorchooser", "tkColorChooser",
                 "tkinter.colorchooser"),
     MovedModule("tkinter_commondialog", "tkCommonDialog",
                 "tkinter.commondialog"),
     MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
     MovedModule("tkinter_font", "tkFont", "tkinter.font"),
     MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
     MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
                 "tkinter.simpledialog"),
     MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
     MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
     MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
     MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
     MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"),
     MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"),
     MovedModule("winreg", "_winreg"),
 ]
 for attr in _moved_attributes:
     setattr(_MovedItems, attr.name, attr)
     if isinstance(attr, MovedModule):
         _importer._add_module(attr, "moves." + attr.name)
 del attr
 
 _MovedItems._moved_attributes = _moved_attributes
 
 moves = _MovedItems(__name__ + ".moves")
 _importer._add_module(moves, "moves")
 
 
 class Module_six_moves_urllib_parse(_LazyModule):
     """Lazy loading of moved objects in six.moves.urllib_parse"""
 
 
 _urllib_parse_moved_attributes = [
     MovedAttribute("ParseResult", "urlparse", "urllib.parse"),
     MovedAttribute("SplitResult", "urlparse", "urllib.parse"),
     MovedAttribute("parse_qs", "urlparse", "urllib.parse"),
     MovedAttribute("parse_qsl", "urlparse", "urllib.parse"),
     MovedAttribute("urldefrag", "urlparse", "urllib.parse"),
     MovedAttribute("urljoin", "urlparse", "urllib.parse"),
     MovedAttribute("urlparse", "urlparse", "urllib.parse"),
     MovedAttribute("urlsplit", "urlparse", "urllib.parse"),
     MovedAttribute("urlunparse", "urlparse", "urllib.parse"),
     MovedAttribute("urlunsplit", "urlparse", "urllib.parse"),
     MovedAttribute("quote", "urllib", "urllib.parse"),
     MovedAttribute("quote_plus", "urllib", "urllib.parse"),
     MovedAttribute("unquote", "urllib", "urllib.parse"),
     MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
     MovedAttribute("urlencode", "urllib", "urllib.parse"),
     MovedAttribute("splitquery", "urllib", "urllib.parse"),
     MovedAttribute("splittag", "urllib", "urllib.parse"),
     MovedAttribute("splituser", "urllib", "urllib.parse"),
     MovedAttribute("uses_fragment", "urlparse", "urllib.parse"),
     MovedAttribute("uses_netloc", "urlparse", "urllib.parse"),
     MovedAttribute("uses_params", "urlparse", "urllib.parse"),
     MovedAttribute("uses_query", "urlparse", "urllib.parse"),
     MovedAttribute("uses_relative", "urlparse", "urllib.parse"),
 ]
 for attr in _urllib_parse_moved_attributes:
     setattr(Module_six_moves_urllib_parse, attr.name, attr)
 del attr
 
 Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes
 
 _importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
                       "moves.urllib_parse", "moves.urllib.parse")
 
 
 class Module_six_moves_urllib_error(_LazyModule):
     """Lazy loading of moved objects in six.moves.urllib_error"""
 
 
 _urllib_error_moved_attributes = [
     MovedAttribute("URLError", "urllib2", "urllib.error"),
     MovedAttribute("HTTPError", "urllib2", "urllib.error"),
     MovedAttribute("ContentTooShortError", "urllib", "urllib.error"),
 ]
 for attr in _urllib_error_moved_attributes:
     setattr(Module_six_moves_urllib_error, attr.name, attr)
 del attr
 
 Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes
 
 _importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
                       "moves.urllib_error", "moves.urllib.error")
 
 
 class Module_six_moves_urllib_request(_LazyModule):
     """Lazy loading of moved objects in six.moves.urllib_request"""
 
 
 _urllib_request_moved_attributes = [
     MovedAttribute("urlopen", "urllib2", "urllib.request"),
     MovedAttribute("install_opener", "urllib2", "urllib.request"),
     MovedAttribute("build_opener", "urllib2", "urllib.request"),
     MovedAttribute("pathname2url", "urllib", "urllib.request"),
     MovedAttribute("url2pathname", "urllib", "urllib.request"),
     MovedAttribute("getproxies", "urllib", "urllib.request"),
     MovedAttribute("Request", "urllib2", "urllib.request"),
     MovedAttribute("OpenerDirector", "urllib2", "urllib.request"),
     MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"),
     MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"),
     MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"),
     MovedAttribute("ProxyHandler", "urllib2", "urllib.request"),
     MovedAttribute("BaseHandler", "urllib2", "urllib.request"),
     MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"),
     MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"),
     MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"),
     MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"),
     MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"),
     MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"),
     MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"),
     MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"),
     MovedAttribute("HTTPHandler", "urllib2", "urllib.request"),
     MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"),
     MovedAttribute("FileHandler", "urllib2", "urllib.request"),
     MovedAttribute("FTPHandler", "urllib2", "urllib.request"),
     MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"),
     MovedAttribute("UnknownHandler", "urllib2", "urllib.request"),
     MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"),
     MovedAttribute("urlretrieve", "urllib", "urllib.request"),
     MovedAttribute("urlcleanup", "urllib", "urllib.request"),
     MovedAttribute("URLopener", "urllib", "urllib.request"),
     MovedAttribute("FancyURLopener", "urllib", "urllib.request"),
     MovedAttribute("proxy_bypass", "urllib", "urllib.request"),
 ]
 for attr in _urllib_request_moved_attributes:
     setattr(Module_six_moves_urllib_request, attr.name, attr)
 del attr
 
 Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes
 
 _importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
                       "moves.urllib_request", "moves.urllib.request")
 
 
 class Module_six_moves_urllib_response(_LazyModule):
     """Lazy loading of moved objects in six.moves.urllib_response"""
 
 
 _urllib_response_moved_attributes = [
     MovedAttribute("addbase", "urllib", "urllib.response"),
     MovedAttribute("addclosehook", "urllib", "urllib.response"),
     MovedAttribute("addinfo", "urllib", "urllib.response"),
     MovedAttribute("addinfourl", "urllib", "urllib.response"),
 ]
 for attr in _urllib_response_moved_attributes:
     setattr(Module_six_moves_urllib_response, attr.name, attr)
 del attr
 
 Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes
 
 _importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
                       "moves.urllib_response", "moves.urllib.response")
 
 
 class Module_six_moves_urllib_robotparser(_LazyModule):
     """Lazy loading of moved objects in six.moves.urllib_robotparser"""
 
 
 _urllib_robotparser_moved_attributes = [
     MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"),
 ]
 for attr in _urllib_robotparser_moved_attributes:
     setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
 del attr
 
 Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes
 
 _importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
                       "moves.urllib_robotparser", "moves.urllib.robotparser")
 
 
 class Module_six_moves_urllib(types.ModuleType):
     """Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
     __path__ = []  # mark as package
     parse = _importer._get_module("moves.urllib_parse")
     error = _importer._get_module("moves.urllib_error")
     request = _importer._get_module("moves.urllib_request")
     response = _importer._get_module("moves.urllib_response")
     robotparser = _importer._get_module("moves.urllib_robotparser")
 
     def __dir__(self):
         return ['parse', 'error', 'request', 'response', 'robotparser']
 
 _importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"),
                       "moves.urllib")
 
 
 def add_move(move):
     """Add an item to six.moves."""
     setattr(_MovedItems, move.name, move)
 
 
 def remove_move(name):
     """Remove item from six.moves."""
     try:
         delattr(_MovedItems, name)
     except AttributeError:
         try:
             del moves.__dict__[name]
         except KeyError:
             raise AttributeError("no such move, %r" % (name,))
 
 
 if PY3:
     _meth_func = "__func__"
     _meth_self = "__self__"
 
     _func_closure = "__closure__"
     _func_code = "__code__"
     _func_defaults = "__defaults__"
     _func_globals = "__globals__"
 else:
     _meth_func = "im_func"
     _meth_self = "im_self"
 
     _func_closure = "func_closure"
     _func_code = "func_code"
     _func_defaults = "func_defaults"
     _func_globals = "func_globals"
 
 
 try:
     advance_iterator = next
 except NameError:
     def advance_iterator(it):
         return it.next()
 next = advance_iterator
 
 
 try:
     callable = callable
 except NameError:
     def callable(obj):
         return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
 
 
 if PY3:
     def get_unbound_function(unbound):
         return unbound
 
     create_bound_method = types.MethodType
 
     Iterator = object
 else:
     def get_unbound_function(unbound):
         return unbound.im_func
 
     def create_bound_method(func, obj):
         return types.MethodType(func, obj, obj.__class__)
 
     class Iterator(object):
 
         def next(self):
             return type(self).__next__(self)
 
     callable = callable
 _add_doc(get_unbound_function,
          """Get the function out of a possibly unbound function""")
 
 
 get_method_function = operator.attrgetter(_meth_func)
 get_method_self = operator.attrgetter(_meth_self)
 get_function_closure = operator.attrgetter(_func_closure)
 get_function_code = operator.attrgetter(_func_code)
 get_function_defaults = operator.attrgetter(_func_defaults)
 get_function_globals = operator.attrgetter(_func_globals)
 
 
 if PY3:
     def iterkeys(d, **kw):
         return iter(d.keys(**kw))
 
     def itervalues(d, **kw):
         return iter(d.values(**kw))
 
     def iteritems(d, **kw):
         return iter(d.items(**kw))
 
     def iterlists(d, **kw):
         return iter(d.lists(**kw))
 
     viewkeys = operator.methodcaller("keys")
 
     viewvalues = operator.methodcaller("values")
 
     viewitems = operator.methodcaller("items")
 else:
     def iterkeys(d, **kw):
         return iter(d.iterkeys(**kw))
 
     def itervalues(d, **kw):
         return iter(d.itervalues(**kw))
 
     def iteritems(d, **kw):
         return iter(d.iteritems(**kw))
 
     def iterlists(d, **kw):
         return iter(d.iterlists(**kw))
 
     viewkeys = operator.methodcaller("viewkeys")
 
     viewvalues = operator.methodcaller("viewvalues")
 
     viewitems = operator.methodcaller("viewitems")
 
 _add_doc(iterkeys, "Return an iterator over the keys of a dictionary.")
 _add_doc(itervalues, "Return an iterator over the values of a dictionary.")
 _add_doc(iteritems,
          "Return an iterator over the (key, value) pairs of a dictionary.")
 _add_doc(iterlists,
          "Return an iterator over the (key, [values]) pairs of a dictionary.")
 
 
 if PY3:
     def b(s):
         return s.encode("latin-1")
     def u(s):
         return s
     unichr = chr
     if sys.version_info[1] <= 1:
         def int2byte(i):
             return bytes((i,))
     else:
         # This is about 2x faster than the implementation above on 3.2+
         int2byte = operator.methodcaller("to_bytes", 1, "big")
     byte2int = operator.itemgetter(0)
     indexbytes = operator.getitem
     iterbytes = iter
     import io
     StringIO = io.StringIO
     BytesIO = io.BytesIO
     _assertCountEqual = "assertCountEqual"
     _assertRaisesRegex = "assertRaisesRegex"
     _assertRegex = "assertRegex"
 else:
     def b(s):
         return s
     # Workaround for standalone backslash
     def u(s):
         return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape")
     unichr = unichr
     int2byte = chr
     def byte2int(bs):
         return ord(bs[0])
     def indexbytes(buf, i):
         return ord(buf[i])
     iterbytes = functools.partial(itertools.imap, ord)
     import StringIO
     StringIO = BytesIO = StringIO.StringIO
     _assertCountEqual = "assertItemsEqual"
     _assertRaisesRegex = "assertRaisesRegexp"
     _assertRegex = "assertRegexpMatches"
 _add_doc(b, """Byte literal""")
 _add_doc(u, """Text literal""")
 
 
 def assertCountEqual(self, *args, **kwargs):
     return getattr(self, _assertCountEqual)(*args, **kwargs)
 
 
 def assertRaisesRegex(self, *args, **kwargs):
     return getattr(self, _assertRaisesRegex)(*args, **kwargs)
 
 
 def assertRegex(self, *args, **kwargs):
     return getattr(self, _assertRegex)(*args, **kwargs)
 
 
 if PY3:
     exec_ = getattr(moves.builtins, "exec")
 
 
     def reraise(tp, value, tb=None):
         if value is None:
             value = tp()
         if value.__traceback__ is not tb:
             raise value.with_traceback(tb)
         raise value
 
 else:
     def exec_(_code_, _globs_=None, _locs_=None):
         """Execute code in a namespace."""
         if _globs_ is None:
             frame = sys._getframe(1)
             _globs_ = frame.f_globals
             if _locs_ is None:
                 _locs_ = frame.f_locals
             del frame
         elif _locs_ is None:
             _locs_ = _globs_
         exec("""exec _code_ in _globs_, _locs_""")
 
 
     exec_("""def reraise(tp, value, tb=None):
     raise tp, value, tb
 """)
 
 
 if sys.version_info[:2] == (3, 2):
     exec_("""def raise_from(value, from_value):
     if from_value is None:
         raise value
     raise value from from_value
 """)
 elif sys.version_info[:2] > (3, 2):
     exec_("""def raise_from(value, from_value):
     raise value from from_value
 """)
 else:
     def raise_from(value, from_value):
         raise value
 
 
 print_ = getattr(moves.builtins, "print", None)
 if print_ is None:
     def print_(*args, **kwargs):
         """The new-style print function for Python 2.4 and 2.5."""
         fp = kwargs.pop("file", sys.stdout)
         if fp is None:
             return
         def write(data):
             if not isinstance(data, basestring):
                 data = str(data)
             # If the file has an encoding, encode unicode with it.
             if (isinstance(fp, file) and
                     isinstance(data, unicode) and
                     fp.encoding is not None):
                 errors = getattr(fp, "errors", None)
                 if errors is None:
                     errors = "strict"
                 data = data.encode(fp.encoding, errors)
             fp.write(data)
         want_unicode = False
         sep = kwargs.pop("sep", None)
         if sep is not None:
             if isinstance(sep, unicode):
                 want_unicode = True
             elif not isinstance(sep, str):
                 raise TypeError("sep must be None or a string")
         end = kwargs.pop("end", None)
         if end is not None:
             if isinstance(end, unicode):
                 want_unicode = True
             elif not isinstance(end, str):
                 raise TypeError("end must be None or a string")
         if kwargs:
             raise TypeError("invalid keyword arguments to print()")
         if not want_unicode:
             for arg in args:
                 if isinstance(arg, unicode):
                     want_unicode = True
                     break
         if want_unicode:
             newline = unicode("\n")
             space = unicode(" ")
         else:
             newline = "\n"
             space = " "
         if sep is None:
             sep = space
         if end is None:
             end = newline
         for i, arg in enumerate(args):
             if i:
                 write(sep)
             write(arg)
         write(end)
 if sys.version_info[:2] < (3, 3):
     _print = print_
     def print_(*args, **kwargs):
         fp = kwargs.get("file", sys.stdout)
         flush = kwargs.pop("flush", False)
         _print(*args, **kwargs)
         if flush and fp is not None:
             fp.flush()
 
 _add_doc(reraise, """Reraise an exception.""")
 
 if sys.version_info[0:2] < (3, 4):
     def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
               updated=functools.WRAPPER_UPDATES):
         def wrapper(f):
             f = functools.wraps(wrapped, assigned, updated)(f)
             f.__wrapped__ = wrapped
             return f
         return wrapper
 else:
     wraps = functools.wraps
 
 def with_metaclass(meta, *bases):
     """Create a base class with a metaclass."""
     # This requires a bit of explanation: the basic idea is to make a dummy
     # metaclass for one level of class instantiation that replaces itself with
     # the actual metaclass.
     class metaclass(meta):
         def __new__(cls, name, this_bases, d):
             return meta(name, bases, d)
     return type.__new__(metaclass, 'temporary_class', (), {})
 
 
 def add_metaclass(metaclass):
     """Class decorator for creating a class with a metaclass."""
     def wrapper(cls):
         orig_vars = cls.__dict__.copy()
         slots = orig_vars.get('__slots__')
         if slots is not None:
             if isinstance(slots, str):
                 slots = [slots]
             for slots_var in slots:
                 orig_vars.pop(slots_var)
         orig_vars.pop('__dict__', None)
         orig_vars.pop('__weakref__', None)
         return metaclass(cls.__name__, cls.__bases__, orig_vars)
     return wrapper
 
 
 def python_2_unicode_compatible(klass):
     """
     A decorator that defines __unicode__ and __str__ methods under Python 2.
     Under Python 3 it does nothing.
 
     To support Python 2 and 3 with a single code base, define a __str__ method
     returning text and apply this decorator to the class.
     """
     if PY2:
         if '__str__' not in klass.__dict__:
             raise ValueError("@python_2_unicode_compatible cannot be applied "
                              "to %s because it doesn't define __str__()." %
                              klass.__name__)
         klass.__unicode__ = klass.__str__
         klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
     return klass
 
 
 # Complete the moves implementation.
 # This code is at the end of this module to speed up module loading.
 # Turn this module into a package.
 __path__ = []  # required for PEP 302 and PEP 451
 __package__ = __name__  # see PEP 366 @ReservedAssignment
 if globals().get("__spec__") is not None:
     __spec__.submodule_search_locations = []  # PEP 451 @UndefinedVariable
 # Remove other six meta path importers, since they cause problems. This can
 # happen if six is removed from sys.modules and then reloaded. (Setuptools does
 # this for some reason.)
 if sys.meta_path:
     for i, importer in enumerate(sys.meta_path):
         # Here's some real nastiness: Another "instance" of the six module might
         # be floating around. Therefore, we can't use isinstance() to check for
         # the six meta path importer, since the other six instance will have
         # inserted an importer with different class.
         if (type(importer).__name__ == "_SixMetaPathImporter" and
                 importer.name == __name__):
             del sys.meta_path[i]
             break
     del i, importer
 # Finally, add the importer to the meta path import hook.
 sys.meta_path.append(_importer)

rhodecode/lib/_vendor/statsd/timer.py

@@ -1,68 +1,68 b''
-from __future__ import absolute_import, division, unicode_literals
+
 
 import functools
 from time import perf_counter as time_now
 
 
 def safe_wraps(wrapper, *args, **kwargs):
     """Safely wraps partial functions."""
     while isinstance(wrapper, functools.partial):
         wrapper = wrapper.func
     return functools.wraps(wrapper, *args, **kwargs)
 
 
 class Timer(object):
     """A context manager/decorator for statsd.timing()."""
 
     def __init__(self, client, stat, rate=1, tags=None, use_decimals=True, auto_send=True):
         self.client = client
         self.stat = stat
         self.rate = rate
         self.tags = tags
         self.ms = None
         self._sent = False
         self._start_time = None
         self.use_decimals = use_decimals
         self.auto_send = auto_send
 
     def __call__(self, f):
         """Thread-safe timing function decorator."""
         @safe_wraps(f)
         def _wrapped(*args, **kwargs):
             start_time = time_now()
             try:
                 return f(*args, **kwargs)
             finally:
                 elapsed_time_ms = 1000.0 * (time_now() - start_time)
                 self.client.timing(self.stat, elapsed_time_ms, self.rate, self.tags, self.use_decimals)
                 self._sent = True
         return _wrapped
 
     def __enter__(self):
         return self.start()
 
     def __exit__(self, typ, value, tb):
         self.stop(send=self.auto_send)
 
     def start(self):
         self.ms = None
         self._sent = False
         self._start_time = time_now()
         return self
 
     def stop(self, send=True):
         if self._start_time is None:
             raise RuntimeError('Timer has not started.')
         dt = time_now() - self._start_time
         self.ms = 1000.0 * dt  # Convert to milliseconds.
         if send:
             self.send()
         return self
 
     def send(self):
         if self.ms is None:
             raise RuntimeError('No data recorded.')
         if self._sent:
             raise RuntimeError('Already sent data.')
         self._sent = True
         self.client.timing(self.stat, self.ms, self.rate, self.tags, self.use_decimals)

rhodecode/lib/diff_match_patch.py

@@ -1,1919 +1,1919 b''
 
-from __future__ import division
+
 
 """Diff Match and Patch
 
 Copyright 2006 Google Inc.
 http://code.google.com/p/google-diff-match-patch/
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
   http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 """
 
 """Functions for diff, match and patch.
 
 Computes the difference between two texts to create a patch.
 Applies the patch onto another text, allowing for errors.
 """
 
 __author__ = '[e-mail] (Neil Fraser)'
 
 import math
 import re
 import sys
 import time
 import urllib
 
 
 class diff_match_patch:
   """Class containing the diff, match and patch methods.
 
   Also contains the behaviour settings.
   """
 
   def __init__(self):
     """Inits a diff_match_patch object with default settings.
     Redefine these in your program to override the defaults.
     """
 
     # Number of seconds to map a diff before giving up (0 for infinity).
     self.Diff_Timeout = 1.0
     # Cost of an empty edit operation in terms of edit characters.
     self.Diff_EditCost = 4
     # At what point is no match declared (0.0 = perfection, 1.0 = very loose).
     self.Match_Threshold = 0.5
     # How far to search for a match (0 = exact location, 1000+ = broad match).
     # A match this many characters away from the expected location will add
     # 1.0 to the score (0.0 is a perfect match).
     self.Match_Distance = 1000
     # When deleting a large block of text (over ~64 characters), how close do
     # the contents have to be to match the expected contents. (0.0 = perfection,
     # 1.0 = very loose).  Note that Match_Threshold controls how closely the
     # end points of a delete need to match.
     self.Patch_DeleteThreshold = 0.5
     # Chunk size for context length.
     self.Patch_Margin = 4
 
     # The number of bits in an int.
     # Python has no maximum, thus to disable patch splitting set to 0.
     # However to avoid long patches in certain pathological cases, use 32.
     # Multiple short patches (using native ints) are much faster than long ones.
     self.Match_MaxBits = 32
 
   #  DIFF FUNCTIONS
 
   # The data structure representing a diff is an array of tuples:
   # [(DIFF_DELETE, "Hello"), (DIFF_INSERT, "Goodbye"), (DIFF_EQUAL, " world.")]
   # which means: delete "Hello", add "Goodbye" and keep " world."
   DIFF_DELETE = -1
   DIFF_INSERT = 1
   DIFF_EQUAL = 0
 
   def diff_main(self, text1, text2, checklines=True, deadline=None):
     """Find the differences between two texts.  Simplifies the problem by
       stripping any common prefix or suffix off the texts before diffing.
 
     Args:
       text1: Old string to be diffed.
       text2: New string to be diffed.
       checklines: Optional speedup flag.  If present and false, then don't run
         a line-level diff first to identify the changed areas.
         Defaults to true, which does a faster, slightly less optimal diff.
       deadline: Optional time when the diff should be complete by.  Used
         internally for recursive calls.  Users should set DiffTimeout instead.
 
     Returns:
       Array of changes.
     """
     # Set a deadline by which time the diff must be complete.
     if deadline is None:
       # Unlike in most languages, Python counts time in seconds.
       if self.Diff_Timeout <= 0:
         deadline = sys.maxint
       else:
         deadline = time.time() + self.Diff_Timeout
 
     # Check for null inputs.
     if text1 is None or text2 is None:
       raise ValueError("Null inputs. (diff_main)")
 
     # Check for equality (speedup).
     if text1 == text2:
       if text1:
         return [(self.DIFF_EQUAL, text1)]
       return []
 
     # Trim off common prefix (speedup).
     commonlength = self.diff_commonPrefix(text1, text2)
     commonprefix = text1[:commonlength]
     text1 = text1[commonlength:]
     text2 = text2[commonlength:]
 
     # Trim off common suffix (speedup).
     commonlength = self.diff_commonSuffix(text1, text2)
     if commonlength == 0:
       commonsuffix = ''
     else:
       commonsuffix = text1[-commonlength:]
       text1 = text1[:-commonlength]
       text2 = text2[:-commonlength]
 
     # Compute the diff on the middle block.
     diffs = self.diff_compute(text1, text2, checklines, deadline)
 
     # Restore the prefix and suffix.
     if commonprefix:
       diffs[:0] = [(self.DIFF_EQUAL, commonprefix)]
     if commonsuffix:
       diffs.append((self.DIFF_EQUAL, commonsuffix))
     self.diff_cleanupMerge(diffs)
     return diffs
 
   def diff_compute(self, text1, text2, checklines, deadline):
     """Find the differences between two texts.  Assumes that the texts do not
       have any common prefix or suffix.
 
     Args:
       text1: Old string to be diffed.
       text2: New string to be diffed.
       checklines: Speedup flag.  If false, then don't run a line-level diff
         first to identify the changed areas.
         If true, then run a faster, slightly less optimal diff.
       deadline: Time when the diff should be complete by.
 
     Returns:
       Array of changes.
     """
     if not text1:
       # Just add some text (speedup).
       return [(self.DIFF_INSERT, text2)]
 
     if not text2:
       # Just delete some text (speedup).
       return [(self.DIFF_DELETE, text1)]
 
     if len(text1) > len(text2):
       (longtext, shorttext) = (text1, text2)
     else:
       (shorttext, longtext) = (text1, text2)
     i = longtext.find(shorttext)
     if i != -1:
       # Shorter text is inside the longer text (speedup).
       diffs = [(self.DIFF_INSERT, longtext[:i]), (self.DIFF_EQUAL, shorttext),
                (self.DIFF_INSERT, longtext[i + len(shorttext):])]
       # Swap insertions for deletions if diff is reversed.
       if len(text1) > len(text2):
         diffs[0] = (self.DIFF_DELETE, diffs[0][1])
         diffs[2] = (self.DIFF_DELETE, diffs[2][1])
       return diffs
 
     if len(shorttext) == 1:
       # Single character string.
       # After the previous speedup, the character can't be an equality.
       return [(self.DIFF_DELETE, text1), (self.DIFF_INSERT, text2)]
 
     # Check to see if the problem can be split in two.
     hm = self.diff_halfMatch(text1, text2)
     if hm:
       # A half-match was found, sort out the return data.
       (text1_a, text1_b, text2_a, text2_b, mid_common) = hm
       # Send both pairs off for separate processing.
       diffs_a = self.diff_main(text1_a, text2_a, checklines, deadline)
       diffs_b = self.diff_main(text1_b, text2_b, checklines, deadline)
       # Merge the results.
       return diffs_a + [(self.DIFF_EQUAL, mid_common)] + diffs_b
 
     if checklines and len(text1) > 100 and len(text2) > 100:
       return self.diff_lineMode(text1, text2, deadline)
 
     return self.diff_bisect(text1, text2, deadline)
 
   def diff_lineMode(self, text1, text2, deadline):
     """Do a quick line-level diff on both strings, then rediff the parts for
       greater accuracy.
       This speedup can produce non-minimal diffs.
 
     Args:
       text1: Old string to be diffed.
       text2: New string to be diffed.
       deadline: Time when the diff should be complete by.
 
     Returns:
       Array of changes.
     """
 
     # Scan the text on a line-by-line basis first.
     (text1, text2, linearray) = self.diff_linesToChars(text1, text2)
 
     diffs = self.diff_main(text1, text2, False, deadline)
 
     # Convert the diff back to original text.
     self.diff_charsToLines(diffs, linearray)
     # Eliminate freak matches (e.g. blank lines)
     self.diff_cleanupSemantic(diffs)
 
     # Rediff any replacement blocks, this time character-by-character.
     # Add a dummy entry at the end.
     diffs.append((self.DIFF_EQUAL, ''))
     pointer = 0
     count_delete = 0
     count_insert = 0
     text_delete = ''
     text_insert = ''
     while pointer < len(diffs):
       if diffs[pointer][0] == self.DIFF_INSERT:
         count_insert += 1
         text_insert += diffs[pointer][1]
       elif diffs[pointer][0] == self.DIFF_DELETE:
         count_delete += 1
         text_delete += diffs[pointer][1]
       elif diffs[pointer][0] == self.DIFF_EQUAL:
         # Upon reaching an equality, check for prior redundancies.
         if count_delete >= 1 and count_insert >= 1:
           # Delete the offending records and add the merged ones.
           a = self.diff_main(text_delete, text_insert, False, deadline)
           diffs[pointer - count_delete - count_insert : pointer] = a
           pointer = pointer - count_delete - count_insert + len(a)
         count_insert = 0
         count_delete = 0
         text_delete = ''
         text_insert = ''
 
       pointer += 1
 
     diffs.pop()  # Remove the dummy entry at the end.
 
     return diffs
 
   def diff_bisect(self, text1, text2, deadline):
     """Find the 'middle snake' of a diff, split the problem in two
       and return the recursively constructed diff.
       See Myers 1986 paper: An O(ND) Difference Algorithm and Its Variations.
 
     Args:
       text1: Old string to be diffed.
       text2: New string to be diffed.
       deadline: Time at which to bail if not yet complete.
 
     Returns:
       Array of diff tuples.
     """
 
     # Cache the text lengths to prevent multiple calls.
     text1_length = len(text1)
     text2_length = len(text2)
     max_d = (text1_length + text2_length + 1) // 2
     v_offset = max_d
     v_length = 2 * max_d
     v1 = [-1] * v_length
     v1[v_offset + 1] = 0
     v2 = v1[:]
     delta = text1_length - text2_length
     # If the total number of characters is odd, then the front path will
     # collide with the reverse path.
     front = (delta % 2 != 0)
     # Offsets for start and end of k loop.
     # Prevents mapping of space beyond the grid.
     k1start = 0
     k1end = 0
     k2start = 0
     k2end = 0
     for d in range(max_d):
       # Bail out if deadline is reached.
       if time.time() > deadline:
         break
 
       # Walk the front path one step.
       for k1 in range(-d + k1start, d + 1 - k1end, 2):
         k1_offset = v_offset + k1
         if k1 == -d or (k1 != d and
             v1[k1_offset - 1] < v1[k1_offset + 1]):
           x1 = v1[k1_offset + 1]
         else:
           x1 = v1[k1_offset - 1] + 1
         y1 = x1 - k1
         while (x1 < text1_length and y1 < text2_length and
                text1[x1] == text2[y1]):
           x1 += 1
           y1 += 1
         v1[k1_offset] = x1
         if x1 > text1_length:
           # Ran off the right of the graph.
           k1end += 2
         elif y1 > text2_length:
           # Ran off the bottom of the graph.
           k1start += 2
         elif front:
           k2_offset = v_offset + delta - k1
           if k2_offset >= 0 and k2_offset < v_length and v2[k2_offset] != -1:
             # Mirror x2 onto top-left coordinate system.
             x2 = text1_length - v2[k2_offset]
             if x1 >= x2:
               # Overlap detected.
               return self.diff_bisectSplit(text1, text2, x1, y1, deadline)
 
       # Walk the reverse path one step.
       for k2 in range(-d + k2start, d + 1 - k2end, 2):
         k2_offset = v_offset + k2
         if k2 == -d or (k2 != d and
             v2[k2_offset - 1] < v2[k2_offset + 1]):
           x2 = v2[k2_offset + 1]
         else:
           x2 = v2[k2_offset - 1] + 1
         y2 = x2 - k2
         while (x2 < text1_length and y2 < text2_length and
                text1[-x2 - 1] == text2[-y2 - 1]):
           x2 += 1
           y2 += 1
         v2[k2_offset] = x2
         if x2 > text1_length:
           # Ran off the left of the graph.
           k2end += 2
         elif y2 > text2_length:
           # Ran off the top of the graph.
           k2start += 2
         elif not front:
           k1_offset = v_offset + delta - k2
           if k1_offset >= 0 and k1_offset < v_length and v1[k1_offset] != -1:
             x1 = v1[k1_offset]
             y1 = v_offset + x1 - k1_offset
             # Mirror x2 onto top-left coordinate system.
             x2 = text1_length - x2
             if x1 >= x2:
               # Overlap detected.
               return self.diff_bisectSplit(text1, text2, x1, y1, deadline)
 
     # Diff took too long and hit the deadline or
     # number of diffs equals number of characters, no commonality at all.
     return [(self.DIFF_DELETE, text1), (self.DIFF_INSERT, text2)]
 
   def diff_bisectSplit(self, text1, text2, x, y, deadline):
     """Given the location of the 'middle snake', split the diff in two parts
     and recurse.
 
     Args:
       text1: Old string to be diffed.
       text2: New string to be diffed.
       x: Index of split point in text1.
       y: Index of split point in text2.
       deadline: Time at which to bail if not yet complete.
 
     Returns:
       Array of diff tuples.
     """
     text1a = text1[:x]
     text2a = text2[:y]
     text1b = text1[x:]
     text2b = text2[y:]
 
     # Compute both diffs serially.
     diffs = self.diff_main(text1a, text2a, False, deadline)
     diffsb = self.diff_main(text1b, text2b, False, deadline)
 
     return diffs + diffsb
 
   def diff_linesToChars(self, text1, text2):
     """Split two texts into an array of strings.  Reduce the texts to a string
     of hashes where each Unicode character represents one line.
 
     Args:
       text1: First string.
       text2: Second string.
 
     Returns:
       Three element tuple, containing the encoded text1, the encoded text2 and
       the array of unique strings.  The zeroth element of the array of unique
       strings is intentionally blank.
     """
     lineArray = []  # e.g. lineArray[4] == "Hello\n"
     lineHash = {}   # e.g. lineHash["Hello\n"] == 4
 
     # "\x00" is a valid character, but various debuggers don't like it.
     # So we'll insert a junk entry to avoid generating a null character.
     lineArray.append('')
 
     def diff_linesToCharsMunge(text):
       """Split a text into an array of strings.  Reduce the texts to a string
       of hashes where each Unicode character represents one line.
       Modifies linearray and linehash through being a closure.
 
       Args:
         text: String to encode.
 
       Returns:
         Encoded string.
       """
       chars = []
       # Walk the text, pulling out a substring for each line.
       # text.split('\n') would would temporarily double our memory footprint.
       # Modifying text would create many large strings to garbage collect.
       lineStart = 0
       lineEnd = -1
       while lineEnd < len(text) - 1:
         lineEnd = text.find('\n', lineStart)
         if lineEnd == -1:
           lineEnd = len(text) - 1
         line = text[lineStart:lineEnd + 1]
         lineStart = lineEnd + 1
 
         if line in lineHash:
           chars.append(unichr(lineHash[line]))
         else:
           lineArray.append(line)
           lineHash[line] = len(lineArray) - 1
           chars.append(unichr(len(lineArray) - 1))
       return "".join(chars)
 
     chars1 = diff_linesToCharsMunge(text1)
     chars2 = diff_linesToCharsMunge(text2)
     return (chars1, chars2, lineArray)
 
   def diff_charsToLines(self, diffs, lineArray):
     """Rehydrate the text in a diff from a string of line hashes to real lines
     of text.
 
     Args:
       diffs: Array of diff tuples.
       lineArray: Array of unique strings.
     """
     for x in range(len(diffs)):
       text = []
       for char in diffs[x][1]:
         text.append(lineArray[ord(char)])
       diffs[x] = (diffs[x][0], "".join(text))
 
   def diff_commonPrefix(self, text1, text2):
     """Determine the common prefix of two strings.
 
     Args:
       text1: First string.
       text2: Second string.
 
     Returns:
       The number of characters common to the start of each string.
     """
     # Quick check for common null cases.
     if not text1 or not text2 or text1[0] != text2[0]:
       return 0
     # Binary search.
     # Performance analysis: http://neil.fraser.name/news/2007/10/09/
     pointermin = 0
     pointermax = min(len(text1), len(text2))
     pointermid = pointermax
     pointerstart = 0
     while pointermin < pointermid:
       if text1[pointerstart:pointermid] == text2[pointerstart:pointermid]:
         pointermin = pointermid
         pointerstart = pointermin
       else:
         pointermax = pointermid
       pointermid = (pointermax - pointermin) // 2 + pointermin
     return pointermid
 
   def diff_commonSuffix(self, text1, text2):
     """Determine the common suffix of two strings.
 
     Args:
       text1: First string.
       text2: Second string.
 
     Returns:
       The number of characters common to the end of each string.
     """
     # Quick check for common null cases.
     if not text1 or not text2 or text1[-1] != text2[-1]:
       return 0
     # Binary search.
     # Performance analysis: http://neil.fraser.name/news/2007/10/09/
     pointermin = 0
     pointermax = min(len(text1), len(text2))
     pointermid = pointermax
     pointerend = 0
     while pointermin < pointermid:
       if (text1[-pointermid:len(text1) - pointerend] ==
           text2[-pointermid:len(text2) - pointerend]):
         pointermin = pointermid
         pointerend = pointermin
       else:
         pointermax = pointermid
       pointermid = (pointermax - pointermin) // 2 + pointermin
     return pointermid
 
   def diff_commonOverlap(self, text1, text2):
     """Determine if the suffix of one string is the prefix of another.
 
     Args:
       text1 First string.
       text2 Second string.
 
     Returns:
       The number of characters common to the end of the first
       string and the start of the second string.
     """
     # Cache the text lengths to prevent multiple calls.
     text1_length = len(text1)
     text2_length = len(text2)
     # Eliminate the null case.
     if text1_length == 0 or text2_length == 0:
       return 0
     # Truncate the longer string.
     if text1_length > text2_length:
       text1 = text1[-text2_length:]
     elif text1_length < text2_length:
       text2 = text2[:text1_length]
     text_length = min(text1_length, text2_length)
     # Quick check for the worst case.
     if text1 == text2:
       return text_length
 
     # Start by looking for a single character match
     # and increase length until no match is found.
     # Performance analysis: http://neil.fraser.name/news/2010/11/04/
     best = 0
     length = 1
     while True:
       pattern = text1[-length:]
       found = text2.find(pattern)
       if found == -1:
         return best
       length += found
       if found == 0 or text1[-length:] == text2[:length]:
         best = length
         length += 1
 
   def diff_halfMatch(self, text1, text2):
     """Do the two texts share a substring which is at least half the length of
     the longer text?
     This speedup can produce non-minimal diffs.
 
     Args:
       text1: First string.
       text2: Second string.
 
     Returns:
       Five element Array, containing the prefix of text1, the suffix of text1,
       the prefix of text2, the suffix of text2 and the common middle.  Or None
       if there was no match.
     """
     if self.Diff_Timeout <= 0:
       # Don't risk returning a non-optimal diff if we have unlimited time.
       return None
     if len(text1) > len(text2):
       (longtext, shorttext) = (text1, text2)
     else:
       (shorttext, longtext) = (text1, text2)
     if len(longtext) < 4 or len(shorttext) * 2 < len(longtext):
       return None  # Pointless.
 
     def diff_halfMatchI(longtext, shorttext, i):
       """Does a substring of shorttext exist within longtext such that the
       substring is at least half the length of longtext?
       Closure, but does not reference any external variables.
 
       Args:
         longtext: Longer string.
         shorttext: Shorter string.
         i: Start index of quarter length substring within longtext.
 
       Returns:
         Five element Array, containing the prefix of longtext, the suffix of
         longtext, the prefix of shorttext, the suffix of shorttext and the
         common middle.  Or None if there was no match.
       """
       seed = longtext[i:i + len(longtext) // 4]
       best_common = ''
       j = shorttext.find(seed)
       while j != -1:
         prefixLength = self.diff_commonPrefix(longtext[i:], shorttext[j:])
         suffixLength = self.diff_commonSuffix(longtext[:i], shorttext[:j])
         if len(best_common) < suffixLength + prefixLength:
           best_common = (shorttext[j - suffixLength:j] +
               shorttext[j:j + prefixLength])
           best_longtext_a = longtext[:i - suffixLength]
           best_longtext_b = longtext[i + prefixLength:]
           best_shorttext_a = shorttext[:j - suffixLength]
           best_shorttext_b = shorttext[j + prefixLength:]
         j = shorttext.find(seed, j + 1)
 
       if len(best_common) * 2 >= len(longtext):
         return (best_longtext_a, best_longtext_b,
                 best_shorttext_a, best_shorttext_b, best_common)
       else:
         return None
 
     # First check if the second quarter is the seed for a half-match.
     hm1 = diff_halfMatchI(longtext, shorttext, (len(longtext) + 3) // 4)
     # Check again based on the third quarter.
     hm2 = diff_halfMatchI(longtext, shorttext, (len(longtext) + 1) // 2)
     if not hm1 and not hm2:
       return None
     elif not hm2:
       hm = hm1
     elif not hm1:
       hm = hm2
     else:
       # Both matched.  Select the longest.
       if len(hm1[4]) > len(hm2[4]):
         hm = hm1
       else:
         hm = hm2
 
     # A half-match was found, sort out the return data.
     if len(text1) > len(text2):
       (text1_a, text1_b, text2_a, text2_b, mid_common) = hm
     else:
       (text2_a, text2_b, text1_a, text1_b, mid_common) = hm
     return (text1_a, text1_b, text2_a, text2_b, mid_common)
 
   def diff_cleanupSemantic(self, diffs):
     """Reduce the number of edits by eliminating semantically trivial
     equalities.
 
     Args:
       diffs: Array of diff tuples.
     """
     changes = False
     equalities = []  # Stack of indices where equalities are found.
     lastequality = None  # Always equal to diffs[equalities[-1]][1]
     pointer = 0  # Index of current position.
     # Number of chars that changed prior to the equality.
     length_insertions1, length_deletions1 = 0, 0
     # Number of chars that changed after the equality.
     length_insertions2, length_deletions2 = 0, 0
     while pointer < len(diffs):
       if diffs[pointer][0] == self.DIFF_EQUAL:  # Equality found.
         equalities.append(pointer)
         length_insertions1, length_insertions2 = length_insertions2, 0
         length_deletions1, length_deletions2 = length_deletions2, 0
         lastequality = diffs[pointer][1]
       else:  # An insertion or deletion.
         if diffs[pointer][0] == self.DIFF_INSERT:
           length_insertions2 += len(diffs[pointer][1])
         else:
           length_deletions2 += len(diffs[pointer][1])
         # Eliminate an equality that is smaller or equal to the edits on both
         # sides of it.
         if (lastequality and (len(lastequality) <=
             max(length_insertions1, length_deletions1)) and
             (len(lastequality) <= max(length_insertions2, length_deletions2))):
           # Duplicate record.
           diffs.insert(equalities[-1], (self.DIFF_DELETE, lastequality))
           # Change second copy to insert.
           diffs[equalities[-1] + 1] = (self.DIFF_INSERT,
               diffs[equalities[-1] + 1][1])
           # Throw away the equality we just deleted.
           equalities.pop()
           # Throw away the previous equality (it needs to be reevaluated).
           if len(equalities):
             equalities.pop()
           if len(equalities):
             pointer = equalities[-1]
           else:
             pointer = -1
           # Reset the counters.
           length_insertions1, length_deletions1 = 0, 0
           length_insertions2, length_deletions2 = 0, 0
           lastequality = None
           changes = True
       pointer += 1
 
     # Normalize the diff.
     if changes:
       self.diff_cleanupMerge(diffs)
     self.diff_cleanupSemanticLossless(diffs)
 
     # Find any overlaps between deletions and insertions.
     # e.g: <del>abcxxx</del><ins>xxxdef</ins>
     #   -> <del>abc</del>xxx<ins>def</ins>
     # e.g: <del>xxxabc</del><ins>defxxx</ins>
     #   -> <ins>def</ins>xxx<del>abc</del>
     # Only extract an overlap if it is as big as the edit ahead or behind it.
     pointer = 1
     while pointer < len(diffs):
       if (diffs[pointer - 1][0] == self.DIFF_DELETE and
           diffs[pointer][0] == self.DIFF_INSERT):
         deletion = diffs[pointer - 1][1]
         insertion = diffs[pointer][1]
         overlap_length1 = self.diff_commonOverlap(deletion, insertion)
         overlap_length2 = self.diff_commonOverlap(insertion, deletion)
         if overlap_length1 >= overlap_length2:
           if (overlap_length1 >= len(deletion) / 2.0 or
               overlap_length1 >= len(insertion) / 2.0):
             # Overlap found.  Insert an equality and trim the surrounding edits.
             diffs.insert(pointer, (self.DIFF_EQUAL,
                                    insertion[:overlap_length1]))
             diffs[pointer - 1] = (self.DIFF_DELETE,
                                   deletion[:len(deletion) - overlap_length1])
             diffs[pointer + 1] = (self.DIFF_INSERT,
                                   insertion[overlap_length1:])
             pointer += 1
         else:
           if (overlap_length2 >= len(deletion) / 2.0 or
               overlap_length2 >= len(insertion) / 2.0):
             # Reverse overlap found.
             # Insert an equality and swap and trim the surrounding edits.
             diffs.insert(pointer, (self.DIFF_EQUAL, deletion[:overlap_length2]))
             diffs[pointer - 1] = (self.DIFF_INSERT,
                                   insertion[:len(insertion) - overlap_length2])
             diffs[pointer + 1] = (self.DIFF_DELETE, deletion[overlap_length2:])
             pointer += 1
         pointer += 1
       pointer += 1
 
   def diff_cleanupSemanticLossless(self, diffs):
     """Look for single edits surrounded on both sides by equalities
     which can be shifted sideways to align the edit to a word boundary.
     e.g: The c<ins>at c</ins>ame. -> The <ins>cat </ins>came.
 
     Args:
       diffs: Array of diff tuples.
     """
 
     def diff_cleanupSemanticScore(one, two):
       """Given two strings, compute a score representing whether the
       internal boundary falls on logical boundaries.
       Scores range from 6 (best) to 0 (worst).
       Closure, but does not reference any external variables.
 
       Args:
         one: First string.
         two: Second string.
 
       Returns:
         The score.
       """
       if not one or not two:
         # Edges are the best.
         return 6
 
       # Each port of this function behaves slightly differently due to
       # subtle differences in each language's definition of things like
       # 'whitespace'.  Since this function's purpose is largely cosmetic,
       # the choice has been made to use each language's native features
       # rather than force total conformity.
       char1 = one[-1]
       char2 = two[0]
       nonAlphaNumeric1 = not char1.isalnum()
       nonAlphaNumeric2 = not char2.isalnum()
       whitespace1 = nonAlphaNumeric1 and char1.isspace()
       whitespace2 = nonAlphaNumeric2 and char2.isspace()
       lineBreak1 = whitespace1 and (char1 == "\r" or char1 == "\n")
       lineBreak2 = whitespace2 and (char2 == "\r" or char2 == "\n")
       blankLine1 = lineBreak1 and self.BLANKLINEEND.search(one)
       blankLine2 = lineBreak2 and self.BLANKLINESTART.match(two)
 
       if blankLine1 or blankLine2:
         # Five points for blank lines.
         return 5
       elif lineBreak1 or lineBreak2:
         # Four points for line breaks.
         return 4
       elif nonAlphaNumeric1 and not whitespace1 and whitespace2:
         # Three points for end of sentences.
         return 3
       elif whitespace1 or whitespace2:
         # Two points for whitespace.
         return 2
       elif nonAlphaNumeric1 or nonAlphaNumeric2:
         # One point for non-alphanumeric.
         return 1
       return 0
 
     pointer = 1
     # Intentionally ignore the first and last element (don't need checking).
     while pointer < len(diffs) - 1:
       if (diffs[pointer - 1][0] == self.DIFF_EQUAL and
           diffs[pointer + 1][0] == self.DIFF_EQUAL):
         # This is a single edit surrounded by equalities.
         equality1 = diffs[pointer - 1][1]
         edit = diffs[pointer][1]
         equality2 = diffs[pointer + 1][1]
 
         # First, shift the edit as far left as possible.
         commonOffset = self.diff_commonSuffix(equality1, edit)
         if commonOffset:
           commonString = edit[-commonOffset:]
           equality1 = equality1[:-commonOffset]
           edit = commonString + edit[:-commonOffset]
           equality2 = commonString + equality2
 
         # Second, step character by character right, looking for the best fit.
         bestEquality1 = equality1
         bestEdit = edit
         bestEquality2 = equality2
         bestScore = (diff_cleanupSemanticScore(equality1, edit) +
             diff_cleanupSemanticScore(edit, equality2))
         while edit and equality2 and edit[0] == equality2[0]:
           equality1 += edit[0]
           edit = edit[1:] + equality2[0]
           equality2 = equality2[1:]
           score = (diff_cleanupSemanticScore(equality1, edit) +
               diff_cleanupSemanticScore(edit, equality2))
           # The >= encourages trailing rather than leading whitespace on edits.
           if score >= bestScore:
             bestScore = score
             bestEquality1 = equality1
             bestEdit = edit
             bestEquality2 = equality2
 
         if diffs[pointer - 1][1] != bestEquality1:
           # We have an improvement, save it back to the diff.
           if bestEquality1:
             diffs[pointer - 1] = (diffs[pointer - 1][0], bestEquality1)
           else:
             del diffs[pointer - 1]
             pointer -= 1
           diffs[pointer] = (diffs[pointer][0], bestEdit)
           if bestEquality2:
             diffs[pointer + 1] = (diffs[pointer + 1][0], bestEquality2)
           else:
             del diffs[pointer + 1]
             pointer -= 1
       pointer += 1
 
   # Define some regex patterns for matching boundaries.
   BLANKLINEEND = re.compile(r"\n\r?\n$");
   BLANKLINESTART = re.compile(r"^\r?\n\r?\n");
 
   def diff_cleanupEfficiency(self, diffs):
     """Reduce the number of edits by eliminating operationally trivial
     equalities.
 
     Args:
       diffs: Array of diff tuples.
     """
     changes = False
     equalities = []  # Stack of indices where equalities are found.
     lastequality = None  # Always equal to diffs[equalities[-1]][1]
     pointer = 0  # Index of current position.
     pre_ins = False  # Is there an insertion operation before the last equality.
     pre_del = False  # Is there a deletion operation before the last equality.
     post_ins = False  # Is there an insertion operation after the last equality.
     post_del = False  # Is there a deletion operation after the last equality.
     while pointer < len(diffs):
       if diffs[pointer][0] == self.DIFF_EQUAL:  # Equality found.
         if (len(diffs[pointer][1]) < self.Diff_EditCost and
             (post_ins or post_del)):
           # Candidate found.
           equalities.append(pointer)
           pre_ins = post_ins
           pre_del = post_del
           lastequality = diffs[pointer][1]
         else:
           # Not a candidate, and can never become one.
           equalities = []
           lastequality = None
 
         post_ins = post_del = False
       else:  # An insertion or deletion.
         if diffs[pointer][0] == self.DIFF_DELETE:
           post_del = True
         else:
           post_ins = True
 
         # Five types to be split:
         # <ins>A</ins><del>B</del>XY<ins>C</ins><del>D</del>
         # <ins>A</ins>X<ins>C</ins><del>D</del>
         # <ins>A</ins><del>B</del>X<ins>C</ins>
         # <ins>A</del>X<ins>C</ins><del>D</del>
         # <ins>A</ins><del>B</del>X<del>C</del>
 
         if lastequality and ((pre_ins and pre_del and post_ins and post_del) or
                              ((len(lastequality) < self.Diff_EditCost / 2) and
                               (pre_ins + pre_del + post_ins + post_del) == 3)):
           # Duplicate record.
           diffs.insert(equalities[-1], (self.DIFF_DELETE, lastequality))
           # Change second copy to insert.
           diffs[equalities[-1] + 1] = (self.DIFF_INSERT,
               diffs[equalities[-1] + 1][1])
           equalities.pop()  # Throw away the equality we just deleted.
           lastequality = None
           if pre_ins and pre_del:
             # No changes made which could affect previous entry, keep going.
             post_ins = post_del = True
             equalities = []
           else:
             if len(equalities):
               equalities.pop()  # Throw away the previous equality.
             if len(equalities):
               pointer = equalities[-1]
             else:
               pointer = -1
             post_ins = post_del = False
           changes = True
       pointer += 1
 
     if changes:
       self.diff_cleanupMerge(diffs)
 
   def diff_cleanupMerge(self, diffs):
     """Reorder and merge like edit sections.  Merge equalities.
     Any edit section can move as long as it doesn't cross an equality.
 
     Args:
       diffs: Array of diff tuples.
     """
     diffs.append((self.DIFF_EQUAL, ''))  # Add a dummy entry at the end.
     pointer = 0
     count_delete = 0
     count_insert = 0
     text_delete = ''
     text_insert = ''
     while pointer < len(diffs):
       if diffs[pointer][0] == self.DIFF_INSERT:
         count_insert += 1
         text_insert += diffs[pointer][1]
         pointer += 1
       elif diffs[pointer][0] == self.DIFF_DELETE:
         count_delete += 1
         text_delete += diffs[pointer][1]
         pointer += 1
       elif diffs[pointer][0] == self.DIFF_EQUAL:
         # Upon reaching an equality, check for prior redundancies.
         if count_delete + count_insert > 1:
           if count_delete != 0 and count_insert != 0:
             # Factor out any common prefixies.
             commonlength = self.diff_commonPrefix(text_insert, text_delete)
             if commonlength != 0:
               x = pointer - count_delete - count_insert - 1
               if x >= 0 and diffs[x][0] == self.DIFF_EQUAL:
                 diffs[x] = (diffs[x][0], diffs[x][1] +
                             text_insert[:commonlength])
               else:
                 diffs.insert(0, (self.DIFF_EQUAL, text_insert[:commonlength]))
                 pointer += 1
               text_insert = text_insert[commonlength:]
               text_delete = text_delete[commonlength:]
             # Factor out any common suffixies.
             commonlength = self.diff_commonSuffix(text_insert, text_delete)
             if commonlength != 0:
               diffs[pointer] = (diffs[pointer][0], text_insert[-commonlength:] +
                   diffs[pointer][1])
               text_insert = text_insert[:-commonlength]
               text_delete = text_delete[:-commonlength]
           # Delete the offending records and add the merged ones.
           if count_delete == 0:
             diffs[pointer - count_insert : pointer] = [
                 (self.DIFF_INSERT, text_insert)]
           elif count_insert == 0:
             diffs[pointer - count_delete : pointer] = [
                 (self.DIFF_DELETE, text_delete)]
           else:
             diffs[pointer - count_delete - count_insert : pointer] = [
                 (self.DIFF_DELETE, text_delete),
                 (self.DIFF_INSERT, text_insert)]
           pointer = pointer - count_delete - count_insert + 1
           if count_delete != 0:
             pointer += 1
           if count_insert != 0:
             pointer += 1
         elif pointer != 0 and diffs[pointer - 1][0] == self.DIFF_EQUAL:
           # Merge this equality with the previous one.
           diffs[pointer - 1] = (diffs[pointer - 1][0],
                                 diffs[pointer - 1][1] + diffs[pointer][1])
           del diffs[pointer]
         else:
           pointer += 1
 
         count_insert = 0
         count_delete = 0
         text_delete = ''
         text_insert = ''
 
     if diffs[-1][1] == '':
       diffs.pop()  # Remove the dummy entry at the end.
 
     # Second pass: look for single edits surrounded on both sides by equalities
     # which can be shifted sideways to eliminate an equality.
     # e.g: A<ins>BA</ins>C -> <ins>AB</ins>AC
     changes = False
     pointer = 1
     # Intentionally ignore the first and last element (don't need checking).
     while pointer < len(diffs) - 1:
       if (diffs[pointer - 1][0] == self.DIFF_EQUAL and
           diffs[pointer + 1][0] == self.DIFF_EQUAL):
         # This is a single edit surrounded by equalities.
         if diffs[pointer][1].endswith(diffs[pointer - 1][1]):
           # Shift the edit over the previous equality.
           diffs[pointer] = (diffs[pointer][0],
               diffs[pointer - 1][1] +
               diffs[pointer][1][:-len(diffs[pointer - 1][1])])
           diffs[pointer + 1] = (diffs[pointer + 1][0],
                                 diffs[pointer - 1][1] + diffs[pointer + 1][1])
           del diffs[pointer - 1]
           changes = True
         elif diffs[pointer][1].startswith(diffs[pointer + 1][1]):
           # Shift the edit over the next equality.
           diffs[pointer - 1] = (diffs[pointer - 1][0],
                                 diffs[pointer - 1][1] + diffs[pointer + 1][1])
           diffs[pointer] = (diffs[pointer][0],
               diffs[pointer][1][len(diffs[pointer + 1][1]):] +
               diffs[pointer + 1][1])
           del diffs[pointer + 1]
           changes = True
       pointer += 1
 
     # If shifts were made, the diff needs reordering and another shift sweep.
     if changes:
       self.diff_cleanupMerge(diffs)
 
   def diff_xIndex(self, diffs, loc):
     """loc is a location in text1, compute and return the equivalent location
     in text2.  e.g. "The cat" vs "The big cat", 1->1, 5->8
 
     Args:
       diffs: Array of diff tuples.
       loc: Location within text1.
 
     Returns:
       Location within text2.
     """
     chars1 = 0
     chars2 = 0
     last_chars1 = 0
     last_chars2 = 0
     for x in range(len(diffs)):
       (op, text) = diffs[x]
       if op != self.DIFF_INSERT:  # Equality or deletion.
         chars1 += len(text)
       if op != self.DIFF_DELETE:  # Equality or insertion.
         chars2 += len(text)
       if chars1 > loc:  # Overshot the location.
         break
       last_chars1 = chars1
       last_chars2 = chars2
 
     if len(diffs) != x and diffs[x][0] == self.DIFF_DELETE:
       # The location was deleted.
       return last_chars2
     # Add the remaining len(character).
     return last_chars2 + (loc - last_chars1)
 
   def diff_prettyHtml(self, diffs):
     """Convert a diff array into a pretty HTML report.
 
     Args:
       diffs: Array of diff tuples.
 
     Returns:
       HTML representation.
     """
     html = []
     for (op, data) in diffs:
       text = (data.replace("&", "&amp;").replace("<", "&lt;")
                  .replace(">", "&gt;").replace("\n", "&para;<br>"))
       if op == self.DIFF_INSERT:
         html.append("<ins style=\"background:#e6ffe6;\">%s</ins>" % text)
       elif op == self.DIFF_DELETE:
         html.append("<del style=\"background:#ffe6e6;\">%s</del>" % text)
       elif op == self.DIFF_EQUAL:
         html.append("<span>%s</span>" % text)
     return "".join(html)
 
   def diff_text1(self, diffs):
     """Compute and return the source text (all equalities and deletions).
 
     Args:
       diffs: Array of diff tuples.
 
     Returns:
       Source text.
     """
     text = []
     for (op, data) in diffs:
       if op != self.DIFF_INSERT:
         text.append(data)
     return "".join(text)
 
   def diff_text2(self, diffs):
     """Compute and return the destination text (all equalities and insertions).
 
     Args:
       diffs: Array of diff tuples.
 
     Returns:
       Destination text.
     """
     text = []
     for (op, data) in diffs:
       if op != self.DIFF_DELETE:
         text.append(data)
     return "".join(text)
 
   def diff_levenshtein(self, diffs):
     """Compute the Levenshtein distance; the number of inserted, deleted or
     substituted characters.
 
     Args:
       diffs: Array of diff tuples.
 
     Returns:
       Number of changes.
     """
     levenshtein = 0
     insertions = 0
     deletions = 0
     for (op, data) in diffs:
       if op == self.DIFF_INSERT:
         insertions += len(data)
       elif op == self.DIFF_DELETE:
         deletions += len(data)
       elif op == self.DIFF_EQUAL:
         # A deletion and an insertion is one substitution.
         levenshtein += max(insertions, deletions)
         insertions = 0
         deletions = 0
     levenshtein += max(insertions, deletions)
     return levenshtein
 
   def diff_toDelta(self, diffs):
     """Crush the diff into an encoded string which describes the operations
     required to transform text1 into text2.
     E.g. =3\t-2\t+ing  -> Keep 3 chars, delete 2 chars, insert 'ing'.
     Operations are tab-separated.  Inserted text is escaped using %xx notation.
 
     Args:
       diffs: Array of diff tuples.
 
     Returns:
       Delta text.
     """
     text = []
     for (op, data) in diffs:
       if op == self.DIFF_INSERT:
         # High ascii will raise UnicodeDecodeError.  Use Unicode instead.
         data = data.encode("utf-8")
         text.append("+" + urllib.quote(data, "!~*'();/?:@&=+$,# "))
       elif op == self.DIFF_DELETE:
         text.append("-%d" % len(data))
       elif op == self.DIFF_EQUAL:
         text.append("=%d" % len(data))
     return "\t".join(text)
 
   def diff_fromDelta(self, text1, delta):
     """Given the original text1, and an encoded string which describes the
     operations required to transform text1 into text2, compute the full diff.
 
     Args:
       text1: Source string for the diff.
       delta: Delta text.
 
     Returns:
       Array of diff tuples.
 
     Raises:
       ValueError: If invalid input.
     """
     if type(delta) == unicode:
       # Deltas should be composed of a subset of ascii chars, Unicode not
       # required.  If this encode raises UnicodeEncodeError, delta is invalid.
       delta = delta.encode("ascii")
     diffs = []
     pointer = 0  # Cursor in text1
     tokens = delta.split("\t")
     for token in tokens:
       if token == "":
         # Blank tokens are ok (from a trailing \t).
         continue
       # Each token begins with a one character parameter which specifies the
       # operation of this token (delete, insert, equality).
       param = token[1:]
       if token[0] == "+":
         param = urllib.unquote(param).decode("utf-8")
         diffs.append((self.DIFF_INSERT, param))
       elif token[0] == "-" or token[0] == "=":
         try:
           n = int(param)
         except ValueError:
           raise ValueError("Invalid number in diff_fromDelta: " + param)
         if n < 0:
           raise ValueError("Negative number in diff_fromDelta: " + param)
         text = text1[pointer : pointer + n]
         pointer += n
         if token[0] == "=":
           diffs.append((self.DIFF_EQUAL, text))
         else:
           diffs.append((self.DIFF_DELETE, text))
       else:
         # Anything else is an error.
         raise ValueError("Invalid diff operation in diff_fromDelta: " +
             token[0])
     if pointer != len(text1):
       raise ValueError(
           "Delta length (%d) does not equal source text length (%d)." %
          (pointer, len(text1)))
     return diffs
 
   #  MATCH FUNCTIONS
 
   def match_main(self, text, pattern, loc):
     """Locate the best instance of 'pattern' in 'text' near 'loc'.
 
     Args:
       text: The text to search.
       pattern: The pattern to search for.
       loc: The location to search around.
 
     Returns:
       Best match index or -1.
     """
     # Check for null inputs.
     if text is None or pattern is None:
       raise ValueError("Null inputs. (match_main)")
 
     loc = max(0, min(loc, len(text)))
     if text == pattern:
       # Shortcut (potentially not guaranteed by the algorithm)
       return 0
     elif not text:
       # Nothing to match.
       return -1
     elif text[loc:loc + len(pattern)] == pattern:
       # Perfect match at the perfect spot!  (Includes case of null pattern)
       return loc
     else:
       # Do a fuzzy compare.
       match = self.match_bitap(text, pattern, loc)
       return match
 
   def match_bitap(self, text, pattern, loc):
     """Locate the best instance of 'pattern' in 'text' near 'loc' using the
     Bitap algorithm.
 
     Args:
       text: The text to search.
       pattern: The pattern to search for.
       loc: The location to search around.
 
     Returns:
       Best match index or -1.
     """
     # Python doesn't have a maxint limit, so ignore this check.
     #if self.Match_MaxBits != 0 and len(pattern) > self.Match_MaxBits:
     #  raise ValueError("Pattern too long for this application.")
 
     # Initialise the alphabet.
     s = self.match_alphabet(pattern)
 
     def match_bitapScore(e, x):
       """Compute and return the score for a match with e errors and x location.
       Accesses loc and pattern through being a closure.
 
       Args:
         e: Number of errors in match.
         x: Location of match.
 
       Returns:
         Overall score for match (0.0 = good, 1.0 = bad).
       """
       accuracy = float(e) / len(pattern)
       proximity = abs(loc - x)
       if not self.Match_Distance:
         # Dodge divide by zero error.
         return proximity and 1.0 or accuracy
       return accuracy + (proximity / float(self.Match_Distance))
 
     # Highest score beyond which we give up.
     score_threshold = self.Match_Threshold
     # Is there a nearby exact match? (speedup)
     best_loc = text.find(pattern, loc)
     if best_loc != -1:
       score_threshold = min(match_bitapScore(0, best_loc), score_threshold)
       # What about in the other direction? (speedup)
       best_loc = text.rfind(pattern, loc + len(pattern))
       if best_loc != -1:
         score_threshold = min(match_bitapScore(0, best_loc), score_threshold)
 
     # Initialise the bit arrays.
     matchmask = 1 << (len(pattern) - 1)
     best_loc = -1
 
     bin_max = len(pattern) + len(text)
     # Empty initialization added to appease pychecker.
     last_rd = None
     for d in range(len(pattern)):
       # Scan for the best match each iteration allows for one more error.
       # Run a binary search to determine how far from 'loc' we can stray at
       # this error level.
       bin_min = 0
       bin_mid = bin_max
       while bin_min < bin_mid:
         if match_bitapScore(d, loc + bin_mid) <= score_threshold:
           bin_min = bin_mid
         else:
           bin_max = bin_mid
         bin_mid = (bin_max - bin_min) // 2 + bin_min
 
       # Use the result from this iteration as the maximum for the next.
       bin_max = bin_mid
       start = max(1, loc - bin_mid + 1)
       finish = min(loc + bin_mid, len(text)) + len(pattern)
 
       rd = [0] * (finish + 2)
       rd[finish + 1] = (1 << d) - 1
       for j in range(finish, start - 1, -1):
         if len(text) <= j - 1:
           # Out of range.
           charMatch = 0
         else:
           charMatch = s.get(text[j - 1], 0)
         if d == 0:  # First pass: exact match.
           rd[j] = ((rd[j + 1] << 1) | 1) & charMatch
         else:  # Subsequent passes: fuzzy match.
           rd[j] = (((rd[j + 1] << 1) | 1) & charMatch) | (
               ((last_rd[j + 1] | last_rd[j]) << 1) | 1) | last_rd[j + 1]
         if rd[j] & matchmask:
           score = match_bitapScore(d, j - 1)
           # This match will almost certainly be better than any existing match.
           # But check anyway.
           if score <= score_threshold:
             # Told you so.
             score_threshold = score
             best_loc = j - 1
             if best_loc > loc:
               # When passing loc, don't exceed our current distance from loc.
               start = max(1, 2 * loc - best_loc)
             else:
               # Already passed loc, downhill from here on in.
               break
       # No hope for a (better) match at greater error levels.
       if match_bitapScore(d + 1, loc) > score_threshold:
         break
       last_rd = rd
     return best_loc
 
   def match_alphabet(self, pattern):
     """Initialise the alphabet for the Bitap algorithm.
 
     Args:
       pattern: The text to encode.
 
     Returns:
       Hash of character locations.
     """
     s = {}
     for char in pattern:
       s[char] = 0
     for i in range(len(pattern)):
       s[pattern[i]] |= 1 << (len(pattern) - i - 1)
     return s
 
   #  PATCH FUNCTIONS
 
   def patch_addContext(self, patch, text):
     """Increase the context until it is unique,
     but don't let the pattern expand beyond Match_MaxBits.
 
     Args:
       patch: The patch to grow.
       text: Source text.
     """
     if len(text) == 0:
       return
     pattern = text[patch.start2 : patch.start2 + patch.length1]
     padding = 0
 
     # Look for the first and last matches of pattern in text.  If two different
     # matches are found, increase the pattern length.
     while (text.find(pattern) != text.rfind(pattern) and (self.Match_MaxBits ==
         0 or len(pattern) < self.Match_MaxBits - self.Patch_Margin -
         self.Patch_Margin)):
       padding += self.Patch_Margin
       pattern = text[max(0, patch.start2 - padding) :
                      patch.start2 + patch.length1 + padding]
     # Add one chunk for good luck.
     padding += self.Patch_Margin
 
     # Add the prefix.
     prefix = text[max(0, patch.start2 - padding) : patch.start2]
     if prefix:
       patch.diffs[:0] = [(self.DIFF_EQUAL, prefix)]
     # Add the suffix.
     suffix = text[patch.start2 + patch.length1 :
                   patch.start2 + patch.length1 + padding]
     if suffix:
       patch.diffs.append((self.DIFF_EQUAL, suffix))
 
     # Roll back the start points.
     patch.start1 -= len(prefix)
     patch.start2 -= len(prefix)
     # Extend lengths.
     patch.length1 += len(prefix) + len(suffix)
     patch.length2 += len(prefix) + len(suffix)
 
   def patch_make(self, a, b=None, c=None):
     """Compute a list of patches to turn text1 into text2.
     Use diffs if provided, otherwise compute it ourselves.
     There are four ways to call this function, depending on what data is
     available to the caller:
     Method 1:
     a = text1, b = text2
     Method 2:
     a = diffs
     Method 3 (optimal):
     a = text1, b = diffs
     Method 4 (deprecated, use method 3):
     a = text1, b = text2, c = diffs
 
     Args:
       a: text1 (methods 1,3,4) or Array of diff tuples for text1 to
           text2 (method 2).
       b: text2 (methods 1,4) or Array of diff tuples for text1 to
           text2 (method 3) or undefined (method 2).
       c: Array of diff tuples for text1 to text2 (method 4) or
           undefined (methods 1,2,3).
 
     Returns:
       Array of Patch objects.
     """
     text1 = None
     diffs = None
     # Note that texts may arrive as 'str' or 'unicode'.
     if isinstance(a, str) and isinstance(b, str) and c is None:
       # Method 1: text1, text2
       # Compute diffs from text1 and text2.
       text1 = a
       diffs = self.diff_main(text1, b, True)
       if len(diffs) > 2:
         self.diff_cleanupSemantic(diffs)
         self.diff_cleanupEfficiency(diffs)
     elif isinstance(a, list) and b is None and c is None:
       # Method 2: diffs
       # Compute text1 from diffs.
       diffs = a
       text1 = self.diff_text1(diffs)
     elif isinstance(a, str) and isinstance(b, list) and c is None:
       # Method 3: text1, diffs
       text1 = a
       diffs = b
     elif (isinstance(a, str) and isinstance(b, str) and
           isinstance(c, list)):
       # Method 4: text1, text2, diffs
       # text2 is not used.
       text1 = a
       diffs = c
     else:
       raise ValueError("Unknown call format to patch_make.")
 
     if not diffs:
       return []  # Get rid of the None case.
     patches = []
     patch = patch_obj()
     char_count1 = 0  # Number of characters into the text1 string.
     char_count2 = 0  # Number of characters into the text2 string.
     prepatch_text = text1  # Recreate the patches to determine context info.
     postpatch_text = text1
     for x in range(len(diffs)):
       (diff_type, diff_text) = diffs[x]
       if len(patch.diffs) == 0 and diff_type != self.DIFF_EQUAL:
         # A new patch starts here.
         patch.start1 = char_count1
         patch.start2 = char_count2
       if diff_type == self.DIFF_INSERT:
         # Insertion
         patch.diffs.append(diffs[x])
         patch.length2 += len(diff_text)
         postpatch_text = (postpatch_text[:char_count2] + diff_text +
                           postpatch_text[char_count2:])
       elif diff_type == self.DIFF_DELETE:
         # Deletion.
         patch.length1 += len(diff_text)
         patch.diffs.append(diffs[x])
         postpatch_text = (postpatch_text[:char_count2] +
                           postpatch_text[char_count2 + len(diff_text):])
       elif (diff_type == self.DIFF_EQUAL and
             len(diff_text) <= 2 * self.Patch_Margin and
             len(patch.diffs) != 0 and len(diffs) != x + 1):
         # Small equality inside a patch.
         patch.diffs.append(diffs[x])
         patch.length1 += len(diff_text)
         patch.length2 += len(diff_text)
 
       if (diff_type == self.DIFF_EQUAL and
           len(diff_text) >= 2 * self.Patch_Margin):
         # Time for a new patch.
         if len(patch.diffs) != 0:
           self.patch_addContext(patch, prepatch_text)
           patches.append(patch)
           patch = patch_obj()
           # Unlike Unidiff, our patch lists have a rolling context.
           # http://code.google.com/p/google-diff-match-patch/wiki/Unidiff
           # Update prepatch text & pos to reflect the application of the
           # just completed patch.
           prepatch_text = postpatch_text
           char_count1 = char_count2
 
       # Update the current character count.
       if diff_type != self.DIFF_INSERT:
         char_count1 += len(diff_text)
       if diff_type != self.DIFF_DELETE:
         char_count2 += len(diff_text)
 
     # Pick up the leftover patch if not empty.
     if len(patch.diffs) != 0:
       self.patch_addContext(patch, prepatch_text)
       patches.append(patch)
     return patches
 
   def patch_deepCopy(self, patches):
     """Given an array of patches, return another array that is identical.
 
     Args:
       patches: Array of Patch objects.
 
     Returns:
       Array of Patch objects.
     """
     patchesCopy = []
     for patch in patches:
       patchCopy = patch_obj()
       # No need to deep copy the tuples since they are immutable.
       patchCopy.diffs = patch.diffs[:]
       patchCopy.start1 = patch.start1
       patchCopy.start2 = patch.start2
       patchCopy.length1 = patch.length1
       patchCopy.length2 = patch.length2
       patchesCopy.append(patchCopy)
     return patchesCopy
 
   def patch_apply(self, patches, text):
     """Merge a set of patches onto the text.  Return a patched text, as well
     as a list of true/false values indicating which patches were applied.
 
     Args:
       patches: Array of Patch objects.
       text: Old text.
 
     Returns:
       Two element Array, containing the new text and an array of boolean values.
     """
     if not patches:
       return (text, [])
 
     # Deep copy the patches so that no changes are made to originals.
     patches = self.patch_deepCopy(patches)
 
     nullPadding = self.patch_addPadding(patches)
     text = nullPadding + text + nullPadding
     self.patch_splitMax(patches)
 
     # delta keeps track of the offset between the expected and actual location
     # of the previous patch.  If there are patches expected at positions 10 and
     # 20, but the first patch was found at 12, delta is 2 and the second patch
     # has an effective expected position of 22.
     delta = 0
     results = []
     for patch in patches:
       expected_loc = patch.start2 + delta
       text1 = self.diff_text1(patch.diffs)
       end_loc = -1
       if len(text1) > self.Match_MaxBits:
         # patch_splitMax will only provide an oversized pattern in the case of
         # a monster delete.
         start_loc = self.match_main(text, text1[:self.Match_MaxBits],
                                     expected_loc)
         if start_loc != -1:
           end_loc = self.match_main(text, text1[-self.Match_MaxBits:],
               expected_loc + len(text1) - self.Match_MaxBits)
           if end_loc == -1 or start_loc >= end_loc:
             # Can't find valid trailing context.  Drop this patch.
             start_loc = -1
       else:
         start_loc = self.match_main(text, text1, expected_loc)
       if start_loc == -1:
         # No match found.  :(
         results.append(False)
         # Subtract the delta for this failed patch from subsequent patches.
         delta -= patch.length2 - patch.length1
       else:
         # Found a match.  :)
         results.append(True)
         delta = start_loc - expected_loc
         if end_loc == -1:
           text2 = text[start_loc : start_loc + len(text1)]
         else:
           text2 = text[start_loc : end_loc + self.Match_MaxBits]
         if text1 == text2:
           # Perfect match, just shove the replacement text in.
           text = (text[:start_loc] + self.diff_text2(patch.diffs) +
                       text[start_loc + len(text1):])
         else:
           # Imperfect match.
           # Run a diff to get a framework of equivalent indices.
           diffs = self.diff_main(text1, text2, False)
           if (len(text1) > self.Match_MaxBits and
               self.diff_levenshtein(diffs) / float(len(text1)) >
               self.Patch_DeleteThreshold):
             # The end points match, but the content is unacceptably bad.
             results[-1] = False
           else:
             self.diff_cleanupSemanticLossless(diffs)
             index1 = 0
             for (op, data) in patch.diffs:
               if op != self.DIFF_EQUAL:
                 index2 = self.diff_xIndex(diffs, index1)
               if op == self.DIFF_INSERT:  # Insertion
                 text = text[:start_loc + index2] + data + text[start_loc +
                                                                index2:]
               elif op == self.DIFF_DELETE:  # Deletion
                 text = text[:start_loc + index2] + text[start_loc +
                     self.diff_xIndex(diffs, index1 + len(data)):]
               if op != self.DIFF_DELETE:
                 index1 += len(data)
     # Strip the padding off.
     text = text[len(nullPadding):-len(nullPadding)]
     return (text, results)
 
   def patch_addPadding(self, patches):
     """Add some padding on text start and end so that edges can match
     something.  Intended to be called only from within patch_apply.
 
     Args:
       patches: Array of Patch objects.
 
     Returns:
       The padding string added to each side.
     """
     paddingLength = self.Patch_Margin
     nullPadding = ""
     for x in range(1, paddingLength + 1):
       nullPadding += chr(x)
 
     # Bump all the patches forward.
     for patch in patches:
       patch.start1 += paddingLength
       patch.start2 += paddingLength
 
     # Add some padding on start of first diff.
     patch = patches[0]
     diffs = patch.diffs
     if not diffs or diffs[0][0] != self.DIFF_EQUAL:
       # Add nullPadding equality.
       diffs.insert(0, (self.DIFF_EQUAL, nullPadding))
       patch.start1 -= paddingLength  # Should be 0.
       patch.start2 -= paddingLength  # Should be 0.
       patch.length1 += paddingLength
       patch.length2 += paddingLength
     elif paddingLength > len(diffs[0][1]):
       # Grow first equality.
       extraLength = paddingLength - len(diffs[0][1])
       newText = nullPadding[len(diffs[0][1]):] + diffs[0][1]
       diffs[0] = (diffs[0][0], newText)
       patch.start1 -= extraLength
       patch.start2 -= extraLength
       patch.length1 += extraLength
       patch.length2 += extraLength
 
     # Add some padding on end of last diff.
     patch = patches[-1]
     diffs = patch.diffs
     if not diffs or diffs[-1][0] != self.DIFF_EQUAL:
       # Add nullPadding equality.
       diffs.append((self.DIFF_EQUAL, nullPadding))
       patch.length1 += paddingLength
       patch.length2 += paddingLength
     elif paddingLength > len(diffs[-1][1]):
       # Grow last equality.
       extraLength = paddingLength - len(diffs[-1][1])
       newText = diffs[-1][1] + nullPadding[:extraLength]
       diffs[-1] = (diffs[-1][0], newText)
       patch.length1 += extraLength
       patch.length2 += extraLength
 
     return nullPadding
 
   def patch_splitMax(self, patches):
     """Look through the patches and break up any which are longer than the
     maximum limit of the match algorithm.
     Intended to be called only from within patch_apply.
 
     Args:
       patches: Array of Patch objects.
     """
     patch_size = self.Match_MaxBits
     if patch_size == 0:
       # Python has the option of not splitting strings due to its ability
       # to handle integers of arbitrary precision.
       return
     for x in range(len(patches)):
       if patches[x].length1 <= patch_size:
         continue
       bigpatch = patches[x]
       # Remove the big old patch.
       del patches[x]
       x -= 1
       start1 = bigpatch.start1
       start2 = bigpatch.start2
       precontext = ''
       while len(bigpatch.diffs) != 0:
         # Create one of several smaller patches.
         patch = patch_obj()
         empty = True
         patch.start1 = start1 - len(precontext)
         patch.start2 = start2 - len(precontext)
         if precontext:
           patch.length1 = patch.length2 = len(precontext)
           patch.diffs.append((self.DIFF_EQUAL, precontext))
 
         while (len(bigpatch.diffs) != 0 and
                patch.length1 < patch_size - self.Patch_Margin):
           (diff_type, diff_text) = bigpatch.diffs[0]
           if diff_type == self.DIFF_INSERT:
             # Insertions are harmless.
             patch.length2 += len(diff_text)
             start2 += len(diff_text)
             patch.diffs.append(bigpatch.diffs.pop(0))
             empty = False
           elif (diff_type == self.DIFF_DELETE and len(patch.diffs) == 1 and
               patch.diffs[0][0] == self.DIFF_EQUAL and
               len(diff_text) > 2 * patch_size):
             # This is a large deletion.  Let it pass in one chunk.
             patch.length1 += len(diff_text)
             start1 += len(diff_text)
             empty = False
             patch.diffs.append((diff_type, diff_text))
             del bigpatch.diffs[0]
           else:
             # Deletion or equality.  Only take as much as we can stomach.
             diff_text = diff_text[:patch_size - patch.length1 -
                                   self.Patch_Margin]
             patch.length1 += len(diff_text)
             start1 += len(diff_text)
             if diff_type == self.DIFF_EQUAL:
               patch.length2 += len(diff_text)
               start2 += len(diff_text)
             else:
               empty = False
 
             patch.diffs.append((diff_type, diff_text))
             if diff_text == bigpatch.diffs[0][1]:
               del bigpatch.diffs[0]
             else:
               bigpatch.diffs[0] = (bigpatch.diffs[0][0],
                                    bigpatch.diffs[0][1][len(diff_text):])
 
         # Compute the head context for the next patch.
         precontext = self.diff_text2(patch.diffs)
         precontext = precontext[-self.Patch_Margin:]
         # Append the end context for this patch.
         postcontext = self.diff_text1(bigpatch.diffs)[:self.Patch_Margin]
         if postcontext:
           patch.length1 += len(postcontext)
           patch.length2 += len(postcontext)
           if len(patch.diffs) != 0 and patch.diffs[-1][0] == self.DIFF_EQUAL:
             patch.diffs[-1] = (self.DIFF_EQUAL, patch.diffs[-1][1] +
                                postcontext)
           else:
             patch.diffs.append((self.DIFF_EQUAL, postcontext))
 
         if not empty:
           x += 1
           patches.insert(x, patch)
 
   def patch_toText(self, patches):
     """Take a list of patches and return a textual representation.
 
     Args:
       patches: Array of Patch objects.
 
     Returns:
       Text representation of patches.
     """
     text = []
     for patch in patches:
       text.append(str(patch))
     return "".join(text)
 
   def patch_fromText(self, textline):
     """Parse a textual representation of patches and return a list of patch
     objects.
 
     Args:
       textline: Text representation of patches.
 
     Returns:
       Array of Patch objects.
 
     Raises:
       ValueError: If invalid input.
     """
     if type(textline) == unicode:
       # Patches should be composed of a subset of ascii chars, Unicode not
       # required.  If this encode raises UnicodeEncodeError, patch is invalid.
       textline = textline.encode("ascii")
     patches = []
     if not textline:
       return patches
     text = textline.split('\n')
     while len(text) != 0:
       m = re.match("^@@ -(\d+),?(\d*) \+(\d+),?(\d*) @@$", text[0])
       if not m:
         raise ValueError("Invalid patch string: " + text[0])
       patch = patch_obj()
       patches.append(patch)
       patch.start1 = int(m.group(1))
       if m.group(2) == '':
         patch.start1 -= 1
         patch.length1 = 1
       elif m.group(2) == '0':
         patch.length1 = 0
       else:
         patch.start1 -= 1
         patch.length1 = int(m.group(2))
 
       patch.start2 = int(m.group(3))
       if m.group(4) == '':
         patch.start2 -= 1
         patch.length2 = 1
       elif m.group(4) == '0':
         patch.length2 = 0
       else:
         patch.start2 -= 1
         patch.length2 = int(m.group(4))
 
       del text[0]
 
       while len(text) != 0:
         if text[0]:
           sign = text[0][0]
         else:
           sign = ''
         line = urllib.unquote(text[0][1:])
         line = line.decode("utf-8")
         if sign == '+':
           # Insertion.
           patch.diffs.append((self.DIFF_INSERT, line))
         elif sign == '-':
           # Deletion.
           patch.diffs.append((self.DIFF_DELETE, line))
         elif sign == ' ':
           # Minor equality.
           patch.diffs.append((self.DIFF_EQUAL, line))
         elif sign == '@':
           # Start of next patch.
           break
         elif sign == '':
           # Blank line?  Whatever.
           pass
         else:
           # WTF?
           raise ValueError("Invalid patch mode: '%s'\n%s" % (sign, line))
         del text[0]
     return patches
 
 
 class patch_obj:
   """Class representing one patch operation.
   """
 
   def __init__(self):
     """Initializes with an empty list of diffs.
     """
     self.diffs = []
     self.start1 = None
     self.start2 = None
     self.length1 = 0
     self.length2 = 0
 
   def __str__(self):
     """Emmulate GNU diff's format.
     Header: @@ -382,8 +481,9 @@
     Indicies are printed as 1-based, not 0-based.
 
     Returns:
       The GNU diff string.
     """
     if self.length1 == 0:
       coords1 = str(self.start1) + ",0"
     elif self.length1 == 1:
       coords1 = str(self.start1 + 1)
     else:
       coords1 = str(self.start1 + 1) + "," + str(self.length1)
     if self.length2 == 0:
       coords2 = str(self.start2) + ",0"
     elif self.length2 == 1:
       coords2 = str(self.start2 + 1)
     else:
       coords2 = str(self.start2 + 1) + "," + str(self.length2)
     text = ["@@ -", coords1, " +", coords2, " @@\n"]
     # Escape the body of the patch with %xx notation.
     for (op, data) in self.diffs:
       if op == diff_match_patch.DIFF_INSERT:
         text.append("+")
       elif op == diff_match_patch.DIFF_DELETE:
         text.append("-")
       elif op == diff_match_patch.DIFF_EQUAL:
         text.append(" ")
       # High ascii will raise UnicodeDecodeError.  Use Unicode instead.
       data = data.encode("utf-8")
       text.append(urllib.quote(data, "!~*'();/?:@&=+$,# ") + "\n")
     return "".join(text)
  No newline at end of file

rhodecode/lib/feedgenerator/feedgenerator.py

@@ -1,446 +1,446 b''
 # Copyright (c) Django Software Foundation and individual contributors.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
 # are permitted provided that the following conditions are met:
 #
 #     1. Redistributions of source code must retain the above copyright notice,
 #        this list of conditions and the following disclaimer.
 #
 #     2. Redistributions in binary form must reproduce the above copyright
 #        notice, this list of conditions and the following disclaimer in the
 #        documentation and/or other materials provided with the distribution.
 #
 #     3. Neither the name of Django nor the names of its contributors may be used
 #        to endorse or promote products derived from this software without
 #        specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 """
 For definitions of the different versions of RSS, see:
 http://web.archive.org/web/20110718035220/http://diveintomark.org/archives/2004/02/04/incompatible-rss
 """
-from __future__ import unicode_literals
+
 
 import datetime
 from StringIO import StringIO
 
 import pytz
 from six.moves.urllib import parse as urlparse
 
 from rhodecode.lib.feedgenerator import datetime_safe
 from rhodecode.lib.feedgenerator.utils import SimplerXMLGenerator, iri_to_uri, force_text
 
 
 #### The following code comes from ``django.utils.feedgenerator`` ####
 
 
 def rfc2822_date(date):
     # We can't use strftime() because it produces locale-dependent results, so
     # we have to map english month and day names manually
     months = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec',)
     days = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')
     # Support datetime objects older than 1900
     date = datetime_safe.new_datetime(date)
     # We do this ourselves to be timezone aware, email.Utils is not tz aware.
     dow = days[date.weekday()]
     month = months[date.month - 1]
     time_str = date.strftime('%s, %%d %s %%Y %%H:%%M:%%S ' % (dow, month))
 
     time_str = time_str.decode('utf-8')
     offset = date.utcoffset()
     # Historically, this function assumes that naive datetimes are in UTC.
     if offset is None:
         return time_str + '-0000'
     else:
         timezone = (offset.days * 24 * 60) + (offset.seconds // 60)
         hour, minute = divmod(timezone, 60)
         return time_str + '%+03d%02d' % (hour, minute)
 
 
 def rfc3339_date(date):
     # Support datetime objects older than 1900
     date = datetime_safe.new_datetime(date)
     time_str = date.strftime('%Y-%m-%dT%H:%M:%S')
 
     time_str = time_str.decode('utf-8')
     offset = date.utcoffset()
     # Historically, this function assumes that naive datetimes are in UTC.
     if offset is None:
         return time_str + 'Z'
     else:
         timezone = (offset.days * 24 * 60) + (offset.seconds // 60)
         hour, minute = divmod(timezone, 60)
         return time_str + '%+03d:%02d' % (hour, minute)
 
 
 def get_tag_uri(url, date):
     """
     Creates a TagURI.
 
     See http://web.archive.org/web/20110514113830/http://diveintomark.org/archives/2004/05/28/howto-atom-id
     """
     bits = urlparse(url)
     d = ''
     if date is not None:
         d = ',%s' % datetime_safe.new_datetime(date).strftime('%Y-%m-%d')
     return 'tag:%s%s:%s/%s' % (bits.hostname, d, bits.path, bits.fragment)
 
 
 class SyndicationFeed(object):
     """Base class for all syndication feeds. Subclasses should provide write()"""
 
     def __init__(self, title, link, description, language=None, author_email=None,
                  author_name=None, author_link=None, subtitle=None, categories=None,
                  feed_url=None, feed_copyright=None, feed_guid=None, ttl=None, **kwargs):
         def to_unicode(s):
             return force_text(s, strings_only=True)
         if categories:
             categories = [force_text(c) for c in categories]
         if ttl is not None:
             # Force ints to unicode
             ttl = force_text(ttl)
         self.feed = {
             'title': to_unicode(title),
             'link': iri_to_uri(link),
             'description': to_unicode(description),
             'language': to_unicode(language),
             'author_email': to_unicode(author_email),
             'author_name': to_unicode(author_name),
             'author_link': iri_to_uri(author_link),
             'subtitle': to_unicode(subtitle),
             'categories': categories or (),
             'feed_url': iri_to_uri(feed_url),
             'feed_copyright': to_unicode(feed_copyright),
             'id': feed_guid or link,
             'ttl': ttl,
         }
         self.feed.update(kwargs)
         self.items = []
 
     def add_item(self, title, link, description, author_email=None,
                  author_name=None, author_link=None, pubdate=None, comments=None,
                  unique_id=None, unique_id_is_permalink=None, enclosure=None,
                  categories=(), item_copyright=None, ttl=None, updateddate=None,
                  enclosures=None, **kwargs):
         """
         Adds an item to the feed. All args are expected to be Python Unicode
         objects except pubdate and updateddate, which are datetime.datetime
         objects, and enclosures, which is an iterable of instances of the
         Enclosure class.
         """
         def to_unicode(s):
             return force_text(s, strings_only=True)
         if categories:
             categories = [to_unicode(c) for c in categories]
         if ttl is not None:
             # Force ints to unicode
             ttl = force_text(ttl)
         if enclosure is None:
             enclosures = [] if enclosures is None else enclosures
 
         item = {
             'title': to_unicode(title),
             'link': iri_to_uri(link),
             'description': to_unicode(description),
             'author_email': to_unicode(author_email),
             'author_name': to_unicode(author_name),
             'author_link': iri_to_uri(author_link),
             'pubdate': pubdate,
             'updateddate': updateddate,
             'comments': to_unicode(comments),
             'unique_id': to_unicode(unique_id),
             'unique_id_is_permalink': unique_id_is_permalink,
             'enclosures': enclosures,
             'categories': categories or (),
             'item_copyright': to_unicode(item_copyright),
             'ttl': ttl,
         }
         item.update(kwargs)
         self.items.append(item)
 
     def num_items(self):
         return len(self.items)
 
     def root_attributes(self):
         """
         Return extra attributes to place on the root (i.e. feed/channel) element.
         Called from write().
         """
         return {}
 
     def add_root_elements(self, handler):
         """
         Add elements in the root (i.e. feed/channel) element. Called
         from write().
         """
         pass
 
     def item_attributes(self, item):
         """
         Return extra attributes to place on each item (i.e. item/entry) element.
         """
         return {}
 
     def add_item_elements(self, handler, item):
         """
         Add elements on each item (i.e. item/entry) element.
         """
         pass
 
     def write(self, outfile, encoding):
         """
         Outputs the feed in the given encoding to outfile, which is a file-like
         object. Subclasses should override this.
         """
         raise NotImplementedError('subclasses of SyndicationFeed must provide a write() method')
 
     def writeString(self, encoding):
         """
         Returns the feed in the given encoding as a string.
         """
         s = StringIO()
         self.write(s, encoding)
         return s.getvalue()
 
     def latest_post_date(self):
         """
         Returns the latest item's pubdate or updateddate. If no items
         have either of these attributes this returns the current UTC date/time.
         """
         latest_date = None
         date_keys = ('updateddate', 'pubdate')
 
         for item in self.items:
             for date_key in date_keys:
                 item_date = item.get(date_key)
                 if item_date:
                     if latest_date is None or item_date > latest_date:
                         latest_date = item_date
 
         # datetime.now(tz=utc) is slower, as documented in django.utils.timezone.now
         return latest_date or datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
 
 
 class Enclosure(object):
     """Represents an RSS enclosure"""
     def __init__(self, url, length, mime_type):
         """All args are expected to be Python Unicode objects"""
         self.length, self.mime_type = length, mime_type
         self.url = iri_to_uri(url)
 
 
 class RssFeed(SyndicationFeed):
     content_type = 'application/rss+xml; charset=utf-8'
 
     def write(self, outfile, encoding):
         handler = SimplerXMLGenerator(outfile, encoding)
         handler.startDocument()
         handler.startElement("rss", self.rss_attributes())
         handler.startElement("channel", self.root_attributes())
         self.add_root_elements(handler)
         self.write_items(handler)
         self.endChannelElement(handler)
         handler.endElement("rss")
 
     def rss_attributes(self):
         return {"version": self._version,
                 "xmlns:atom": "http://www.w3.org/2005/Atom"}
 
     def write_items(self, handler):
         for item in self.items:
             handler.startElement('item', self.item_attributes(item))
             self.add_item_elements(handler, item)
             handler.endElement("item")
 
     def add_root_elements(self, handler):
         handler.addQuickElement("title", self.feed['title'])
         handler.addQuickElement("link", self.feed['link'])
         handler.addQuickElement("description", self.feed['description'])
         if self.feed['feed_url'] is not None:
             handler.addQuickElement("atom:link", None, {"rel": "self", "href": self.feed['feed_url']})
         if self.feed['language'] is not None:
             handler.addQuickElement("language", self.feed['language'])
         for cat in self.feed['categories']:
             handler.addQuickElement("category", cat)
         if self.feed['feed_copyright'] is not None:
             handler.addQuickElement("copyright", self.feed['feed_copyright'])
         handler.addQuickElement("lastBuildDate", rfc2822_date(self.latest_post_date()))
         if self.feed['ttl'] is not None:
             handler.addQuickElement("ttl", self.feed['ttl'])
 
     def endChannelElement(self, handler):
         handler.endElement("channel")
 
 
 class RssUserland091Feed(RssFeed):
     _version = "0.91"
 
     def add_item_elements(self, handler, item):
         handler.addQuickElement("title", item['title'])
         handler.addQuickElement("link", item['link'])
         if item['description'] is not None:
             handler.addQuickElement("description", item['description'])
 
 
 class Rss201rev2Feed(RssFeed):
     # Spec: http://blogs.law.harvard.edu/tech/rss
     _version = "2.0"
 
     def add_item_elements(self, handler, item):
         handler.addQuickElement("title", item['title'])
         handler.addQuickElement("link", item['link'])
         if item['description'] is not None:
             handler.addQuickElement("description", item['description'])
 
         # Author information.
         if item["author_name"] and item["author_email"]:
             handler.addQuickElement("author", "%s (%s)" % (item['author_email'], item['author_name']))
         elif item["author_email"]:
             handler.addQuickElement("author", item["author_email"])
         elif item["author_name"]:
             handler.addQuickElement(
                 "dc:creator", item["author_name"], {"xmlns:dc": "http://purl.org/dc/elements/1.1/"}
             )
 
         if item['pubdate'] is not None:
             handler.addQuickElement("pubDate", rfc2822_date(item['pubdate']))
         if item['comments'] is not None:
             handler.addQuickElement("comments", item['comments'])
         if item['unique_id'] is not None:
             guid_attrs = {}
             if isinstance(item.get('unique_id_is_permalink'), bool):
                 guid_attrs['isPermaLink'] = str(item['unique_id_is_permalink']).lower()
             handler.addQuickElement("guid", item['unique_id'], guid_attrs)
         if item['ttl'] is not None:
             handler.addQuickElement("ttl", item['ttl'])
 
         # Enclosure.
         if item['enclosures']:
             enclosures = list(item['enclosures'])
             if len(enclosures) > 1:
                 raise ValueError(
                     "RSS feed items may only have one enclosure, see "
                     "http://www.rssboard.org/rss-profile#element-channel-item-enclosure"
                 )
             enclosure = enclosures[0]
             handler.addQuickElement('enclosure', '', {
                 'url': enclosure.url,
                 'length': enclosure.length,
                 'type': enclosure.mime_type,
             })
 
         # Categories.
         for cat in item['categories']:
             handler.addQuickElement("category", cat)
 
 
 class Atom1Feed(SyndicationFeed):
     # Spec: https://tools.ietf.org/html/rfc4287
     content_type = 'application/atom+xml; charset=utf-8'
     ns = "http://www.w3.org/2005/Atom"
 
     def write(self, outfile, encoding):
         handler = SimplerXMLGenerator(outfile, encoding)
         handler.startDocument()
         handler.startElement('feed', self.root_attributes())
         self.add_root_elements(handler)
         self.write_items(handler)
         handler.endElement("feed")
 
     def root_attributes(self):
         if self.feed['language'] is not None:
             return {"xmlns": self.ns, "xml:lang": self.feed['language']}
         else:
             return {"xmlns": self.ns}
 
     def add_root_elements(self, handler):
         handler.addQuickElement("title", self.feed['title'])
         handler.addQuickElement("link", "", {"rel": "alternate", "href": self.feed['link']})
         if self.feed['feed_url'] is not None:
             handler.addQuickElement("link", "", {"rel": "self", "href": self.feed['feed_url']})
         handler.addQuickElement("id", self.feed['id'])
         handler.addQuickElement("updated", rfc3339_date(self.latest_post_date()))
         if self.feed['author_name'] is not None:
             handler.startElement("author", {})
             handler.addQuickElement("name", self.feed['author_name'])
             if self.feed['author_email'] is not None:
                 handler.addQuickElement("email", self.feed['author_email'])
             if self.feed['author_link'] is not None:
                 handler.addQuickElement("uri", self.feed['author_link'])
             handler.endElement("author")
         if self.feed['subtitle'] is not None:
             handler.addQuickElement("subtitle", self.feed['subtitle'])
         for cat in self.feed['categories']:
             handler.addQuickElement("category", "", {"term": cat})
         if self.feed['feed_copyright'] is not None:
             handler.addQuickElement("rights", self.feed['feed_copyright'])
 
     def write_items(self, handler):
         for item in self.items:
             handler.startElement("entry", self.item_attributes(item))
             self.add_item_elements(handler, item)
             handler.endElement("entry")
 
     def add_item_elements(self, handler, item):
         handler.addQuickElement("title", item['title'])
         handler.addQuickElement("link", "", {"href": item['link'], "rel": "alternate"})
 
         if item['pubdate'] is not None:
             handler.addQuickElement('published', rfc3339_date(item['pubdate']))
 
         if item['updateddate'] is not None:
             handler.addQuickElement('updated', rfc3339_date(item['updateddate']))
 
         # Author information.
         if item['author_name'] is not None:
             handler.startElement("author", {})
             handler.addQuickElement("name", item['author_name'])
             if item['author_email'] is not None:
                 handler.addQuickElement("email", item['author_email'])
             if item['author_link'] is not None:
                 handler.addQuickElement("uri", item['author_link'])
             handler.endElement("author")
 
         # Unique ID.
         if item['unique_id'] is not None:
             unique_id = item['unique_id']
         else:
             unique_id = get_tag_uri(item['link'], item['pubdate'])
         handler.addQuickElement("id", unique_id)
 
         # Summary.
         if item['description'] is not None:
             handler.addQuickElement("summary", item['description'], {"type": "html"})
 
         # Enclosures.
         for enclosure in item['enclosures']:
             handler.addQuickElement('link', '', {
                 'rel': 'enclosure',
                 'href': enclosure.url,
                 'length': enclosure.length,
                 'type': enclosure.mime_type,
             })
 
         # Categories.
         for cat in item['categories']:
             handler.addQuickElement("category", "", {"term": cat})
 
         # Rights.
         if item['item_copyright'] is not None:
             handler.addQuickElement("rights", item['item_copyright'])
 
 
 # This isolates the decision of what the system default is, so calling code can
 # do "feedgenerator.DefaultFeed" instead of "feedgenerator.Rss201rev2Feed".
 DefaultFeed = Rss201rev2Feed
  No newline at end of file

rhodecode/lib/index/whoosh.py

@@ -1,311 +1,311 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Index schema for RhodeCode
 """
 
-from __future__ import absolute_import
+
 import os
 import re
 import logging
 
 from whoosh import query as query_lib
 from whoosh.highlight import HtmlFormatter, ContextFragmenter
 from whoosh.index import create_in, open_dir, exists_in, EmptyIndexError
 from whoosh.qparser import QueryParser, QueryParserError
 
 import rhodecode.lib.helpers as h
 from rhodecode.lib.index import BaseSearcher
 from rhodecode.lib.utils2 import safe_unicode
 
 log = logging.getLogger(__name__)
 
 
 try:
     # we first try to import from rhodecode tools, fallback to copies if
     # we're unable to
     from rhodecode_tools.lib.fts_index.whoosh_schema import (
         ANALYZER, FILE_INDEX_NAME, FILE_SCHEMA, COMMIT_INDEX_NAME,
         COMMIT_SCHEMA)
 except ImportError:
     log.warning('rhodecode_tools schema not available, doing a fallback '
                 'import from `rhodecode.lib.index.whoosh_fallback_schema`')
     from rhodecode.lib.index.whoosh_fallback_schema import (
         ANALYZER, FILE_INDEX_NAME, FILE_SCHEMA, COMMIT_INDEX_NAME,
         COMMIT_SCHEMA)
 
 
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')
 FRAGMENTER = ContextFragmenter(200)
 
 log = logging.getLogger(__name__)
 
 
 class WhooshSearcher(BaseSearcher):
     # this also shows in UI
     query_lang_doc = 'http://whoosh.readthedocs.io/en/latest/querylang.html'
     name = 'whoosh'
 
     def __init__(self, config):
         super(Searcher, self).__init__()
         self.config = config
         if not os.path.isdir(self.config['location']):
             os.makedirs(self.config['location'])
 
         opener = create_in
         if exists_in(self.config['location'], indexname=FILE_INDEX_NAME):
             opener = open_dir
         file_index = opener(self.config['location'], schema=FILE_SCHEMA,
                             indexname=FILE_INDEX_NAME)
 
         opener = create_in
         if exists_in(self.config['location'], indexname=COMMIT_INDEX_NAME):
             opener = open_dir
         changeset_index = opener(self.config['location'], schema=COMMIT_SCHEMA,
                                  indexname=COMMIT_INDEX_NAME)
 
         self.commit_schema = COMMIT_SCHEMA
         self.commit_index = changeset_index
         self.file_schema = FILE_SCHEMA
         self.file_index = file_index
         self.searcher = None
 
     def cleanup(self):
         if self.searcher:
             self.searcher.close()
 
     def _extend_query(self, query):
         hashes = re.compile('([0-9a-f]{5,40})').findall(query)
         if hashes:
             hashes_or_query = ' OR '.join('commit_id:%s*' % h for h in hashes)
             query = u'(%s) OR %s' % (query, hashes_or_query)
         return query
 
     def sort_def(self, search_type, direction, sort_field):
 
         if search_type == 'commit':
             field_defs = {
                 'message': 'message',
                 'date': 'date',
                 'author_email': 'author',
             }
         elif search_type == 'path':
             field_defs = {
                 'file': 'path',
                 'size': 'size',
                 'lines': 'lines',
             }
         elif search_type == 'content':
             # NOTE(dan): content doesn't support any sorting
             field_defs = {}
         else:
             return ''
 
         if sort_field in field_defs:
             return field_defs[sort_field]
 
     def search(self, query, document_type, search_user,
                repo_name=None, repo_group_name=None,
                requested_page=1, page_limit=10, sort=None, raise_on_exc=True):
 
         original_query = query
         query = self._extend_query(query)
 
         log.debug(u'QUERY: %s on %s', query, document_type)
         result = {
             'results': [],
             'count': 0,
             'error': None,
             'runtime': 0
         }
         search_type, index_name, schema_defn = self._prepare_for_search(
             document_type)
         self._init_searcher(index_name)
         try:
             qp = QueryParser(search_type, schema=schema_defn)
             allowed_repos_filter = self._get_repo_filter(
                 search_user, repo_name)
             try:
                 query = qp.parse(safe_unicode(query))
                 log.debug('query: %s (%s)', query, repr(query))
 
                 reverse, sorted_by = False, None
                 direction, sort_field = self.get_sort(search_type, sort)
                 if sort_field:
                     sort_definition = self.sort_def(search_type, direction, sort_field)
                     if sort_definition:
                         sorted_by = sort_definition
                         if direction == Searcher.DIRECTION_DESC:
                             reverse = True
                         if direction == Searcher.DIRECTION_ASC:
                             reverse = False
 
                 whoosh_results = self.searcher.search(
                     query, filter=allowed_repos_filter, limit=None,
                     sortedby=sorted_by, reverse=reverse)
 
                 # fixes for 32k limit that whoosh uses for highlight
                 whoosh_results.fragmenter.charlimit = None
                 res_ln = whoosh_results.scored_length()
                 result['runtime'] = whoosh_results.runtime
                 result['count'] = res_ln
                 result['results'] = WhooshResultWrapper(
                     search_type, res_ln, whoosh_results)
 
             except QueryParserError:
                 result['error'] = 'Invalid search query. Try quoting it.'
         except (EmptyIndexError, IOError, OSError):
             msg = 'There is no index to search in. Please run whoosh indexer'
             log.exception(msg)
             result['error'] = msg
         except Exception:
             msg = 'An error occurred during this search operation'
             log.exception(msg)
             result['error'] = msg
 
         return result
 
     def statistics(self, translator):
         _ = translator
         stats = [
             {'key': _('Index Type'), 'value': 'Whoosh'},
             {'sep': True},
 
             {'key': _('File Index'), 'value': str(self.file_index)},
             {'key': _('Indexed documents'), 'value': self.file_index.doc_count()},
             {'key': _('Last update'), 'value': h.time_to_datetime(self.file_index.last_modified())},
 
             {'sep': True},
 
             {'key': _('Commit index'), 'value': str(self.commit_index)},
             {'key': _('Indexed documents'), 'value': str(self.commit_index.doc_count())},
             {'key': _('Last update'), 'value': h.time_to_datetime(self.commit_index.last_modified())}
         ]
         return stats
 
     def _get_repo_filter(self, auth_user, repo_name):
 
         allowed_to_search = [
             repo for repo, perm in
             auth_user.permissions['repositories'].items()
             if perm != 'repository.none']
 
         if repo_name:
             repo_filter = [query_lib.Term('repository', repo_name)]
 
         elif 'hg.admin' in auth_user.permissions.get('global', []):
             return None
 
         else:
             repo_filter = [query_lib.Term('repository', _rn)
                            for _rn in allowed_to_search]
             # in case we're not allowed to search anywhere, it's a trick
             # to tell whoosh we're filtering, on ALL results
             repo_filter = repo_filter or [query_lib.Term('repository', '')]
 
         return query_lib.Or(repo_filter)
 
     def _prepare_for_search(self, cur_type):
         search_type = {
             'content': 'content',
             'commit': 'message',
             'path': 'path',
             'repository': 'repository'
         }.get(cur_type, 'content')
 
         index_name = {
             'content': FILE_INDEX_NAME,
             'commit': COMMIT_INDEX_NAME,
             'path': FILE_INDEX_NAME
         }.get(cur_type, FILE_INDEX_NAME)
 
         schema_defn = {
             'content': self.file_schema,
             'commit': self.commit_schema,
             'path': self.file_schema
         }.get(cur_type, self.file_schema)
 
         log.debug('IDX: %s', index_name)
         log.debug('SCHEMA: %s', schema_defn)
         return search_type, index_name, schema_defn
 
     def _init_searcher(self, index_name):
         idx = open_dir(self.config['location'], indexname=index_name)
         self.searcher = idx.searcher()
         return self.searcher
 
 
 Searcher = WhooshSearcher
 
 
 class WhooshResultWrapper(object):
     def __init__(self, search_type, total_hits, results):
         self.search_type = search_type
         self.results = results
         self.total_hits = total_hits
 
     def __str__(self):
         return '<%s at %s>' % (self.__class__.__name__, len(self))
 
     def __repr__(self):
         return self.__str__()
 
     def __len__(self):
         return self.total_hits
 
     def __iter__(self):
         """
         Allows Iteration over results,and lazy generate content
 
         *Requires* implementation of ``__getitem__`` method.
         """
         for hit in self.results:
             yield self.get_full_content(hit)
 
     def __getitem__(self, key):
         """
         Slicing of resultWrapper
         """
         i, j = key.start, key.stop
         for hit in self.results[i:j]:
             yield self.get_full_content(hit)
 
     def get_full_content(self, hit):
         # TODO: marcink: this feels like an overkill, there's a lot of data
         # inside hit object, and we don't need all
         res = dict(hit)
         # elastic search uses that, we set it empty so it fallbacks to regular HL logic
         res['content_highlight'] = ''
 
         f_path = ''  # pragma: no cover
         if self.search_type in ['content', 'path']:
             f_path = res['path'][len(res['repository']):]
             f_path = f_path.lstrip(os.sep)
 
         if self.search_type == 'content':
             res.update({'content_short_hl': hit.highlights('content'),
                         'f_path': f_path})
         elif self.search_type == 'path':
             res.update({'f_path': f_path})
         elif self.search_type == 'message':
             res.update({'message_hl': hit.highlights('message')})
 
         return res

rhodecode/lib/index/whoosh_fallback_schema.py

@@ -1,75 +1,75 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Whoosh fallback schema for RhodeCode in case rhodecode_tools defined one is
 not available
 """
 
-from __future__ import absolute_import
+
 
 from whoosh.analysis import RegexTokenizer, LowercaseFilter
 from whoosh.formats import Characters
 from whoosh.fields import (
     TEXT, ID, STORED, NUMERIC, BOOLEAN, Schema, FieldType, DATETIME)
 
 # CUSTOM ANALYZER wordsplit + lowercase filter for case insensitive search
 ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()
 
 # FILE INDEX SCHEMA DEFINITION
 FILE_INDEX_NAME = 'FILE_INDEX'
 FILE_SCHEMA = Schema(
     fileid=ID(unique=True),  # Path
     repository=ID(stored=True),
     repository_id=NUMERIC(unique=True, stored=True),  # Numeric id of repo
     repo_name=TEXT(stored=True),
     owner=TEXT(),
     path=TEXT(stored=True),
     content=FieldType(format=Characters(), analyzer=ANALYZER,
                       scorable=True, stored=True),
     modtime=STORED(),
     md5=STORED(),
     extension=ID(stored=True),
     commit_id=TEXT(stored=True),
 
     size=NUMERIC(int, 64, signed=False, stored=True),
     mimetype=TEXT(stored=True),
     lines=NUMERIC(int, 64, signed=False, stored=True),
 )
 
 
 # COMMIT INDEX SCHEMA
 COMMIT_INDEX_NAME = 'COMMIT_INDEX'
 COMMIT_SCHEMA = Schema(
     commit_id=ID(unique=True, stored=True),
     repository=ID(unique=True, stored=True),
     repository_id=NUMERIC(unique=True, stored=True),
     commit_idx=NUMERIC(stored=True, sortable=True),
     commit_idx_sort=ID(),
     date=NUMERIC(int, 64, signed=False, stored=True, sortable=True),
     owner=TEXT(stored=True),
     author=TEXT(stored=True),
     message=FieldType(format=Characters(), analyzer=ANALYZER,
                       scorable=True, stored=True),
     parents=TEXT(stored=True),
     added=TEXT(stored=True),  # space separated names of added files
     removed=TEXT(stored=True),  # space separated names of removed files
     changed=TEXT(stored=True),  # space separated names of changed files
 )

rhodecode/lib/vcs/utils/helpers.py

@@ -1,161 +1,161 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Utilities aimed to help achieve mostly basic tasks.
 """
 
 
-from __future__ import division
+
 
 import re
 import os
 import time
 import datetime
 import logging
 
 from rhodecode.lib.vcs.conf import settings
 from rhodecode.lib.vcs.exceptions import VCSError, VCSBackendNotSupportedError
 
 
 log = logging.getLogger(__name__)
 
 
 def get_scm(path):
     """
     Returns one of alias from ``ALIASES`` (in order of precedence same as
     shortcuts given in ``ALIASES``) and working dir path for the given
     argument. If no scm-specific directory is found or more than one scm is
     found at that directory, ``VCSError`` is raised.
     """
     if not os.path.isdir(path):
         raise VCSError("Given path %s is not a directory" % path)
 
     found_scms = [(scm, path) for scm in get_scms_for_path(path)]
 
     if len(found_scms) > 1:
         found = ', '.join((x[0] for x in found_scms))
         raise VCSError(
             'More than one [%s] scm found at given path %s' % (found, path))
 
     if len(found_scms) is 0:
         raise VCSError('No scm found at given path %s' % path)
 
     return found_scms[0]
 
 
 def get_scm_backend(backend_type):
     from rhodecode.lib.vcs.backends import get_backend
     return get_backend(backend_type)
 
 
 def get_scms_for_path(path):
     """
     Returns all scm's found at the given path. If no scm is recognized
     - empty list is returned.
 
     :param path: path to directory which should be checked. May be callable.
 
     :raises VCSError: if given ``path`` is not a directory
     """
     from rhodecode.lib.vcs.backends import get_backend
     if hasattr(path, '__call__'):
         path = path()
     if not os.path.isdir(path):
         raise VCSError("Given path %r is not a directory" % path)
 
     result = []
     for key in settings.available_aliases():
         try:
             backend = get_backend(key)
         except VCSBackendNotSupportedError:
             log.warning('VCSBackendNotSupportedError: %s not supported', key)
             continue
         if backend.is_valid_repository(path):
             result.append(key)
     return result
 
 
 def parse_datetime(text):
     """
     Parses given text and returns ``datetime.datetime`` instance or raises
     ``ValueError``.
 
     :param text: string of desired date/datetime or something more verbose,
       like *yesterday*, *2weeks 3days*, etc.
     """
     if not text:
         raise ValueError('Wrong date: "%s"' % text)
 
     if isinstance(text, datetime.datetime):
         return text
 
     # we limit a format to no include microseconds e.g 2017-10-17t17:48:23.XXXX
     text = text.strip().lower()[:19]
 
     input_formats = (
         '%Y-%m-%d %H:%M:%S',
         '%Y-%m-%dt%H:%M:%S',
         '%Y-%m-%d %H:%M',
         '%Y-%m-%dt%H:%M',
         '%Y-%m-%d',
         '%m/%d/%Y %H:%M:%S',
         '%m/%d/%Yt%H:%M:%S',
         '%m/%d/%Y %H:%M',
         '%m/%d/%Yt%H:%M',
         '%m/%d/%Y',
         '%m/%d/%y %H:%M:%S',
         '%m/%d/%yt%H:%M:%S',
         '%m/%d/%y %H:%M',
         '%m/%d/%yt%H:%M',
         '%m/%d/%y',
     )
     for format_def in input_formats:
         try:
             return datetime.datetime(*time.strptime(text, format_def)[:6])
         except ValueError:
             pass
 
     # Try descriptive texts
     if text == 'tomorrow':
         future = datetime.datetime.now() + datetime.timedelta(days=1)
         args = future.timetuple()[:3] + (23, 59, 59)
         return datetime.datetime(*args)
     elif text == 'today':
         return datetime.datetime(*datetime.datetime.today().timetuple()[:3])
     elif text == 'now':
         return datetime.datetime.now()
     elif text == 'yesterday':
         past = datetime.datetime.now() - datetime.timedelta(days=1)
         return datetime.datetime(*past.timetuple()[:3])
     else:
         days = 0
         matched = re.match(
             r'^((?P<weeks>\d+) ?w(eeks?)?)? ?((?P<days>\d+) ?d(ays?)?)?$', text)
         if matched:
             groupdict = matched.groupdict()
             if groupdict['days']:
                 days += int(matched.groupdict()['days'])
             if groupdict['weeks']:
                 days += int(matched.groupdict()['weeks']) * 7
             past = datetime.datetime.now() - datetime.timedelta(days=days)
             return datetime.datetime(*past.timetuple()[:3])
 
     raise ValueError('Wrong date: "%s"' % text)

rhodecode/tests/lib/test_search.py

@@ -1,84 +1,84 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-from __future__ import unicode_literals
+
 
 import time
 
 from whoosh import index
 import mock
 import pytest
 
 import rhodecode
 from rhodecode.lib.auth import AuthUser
 from rhodecode.lib.index import whoosh, searcher_from_config
 
 
 @pytest.mark.parametrize("name_suffix", [
     "",
     "UpPeRcAsE",
 ])
 def test_search_finds_results(
         tmpdir, backend_random, user_regular, name_suffix):
     repo = backend_random.create_repo(name_suffix=name_suffix)
 
     search_location = tmpdir.strpath
     create_commit_index_with_one_document(
         search_location, repo_name=repo.repo_name)
 
     auth_user = AuthUser(user_id=user_regular.user_id)
     with mock.patch.dict(rhodecode.CONFIG,
                          {'search.location': search_location}):
         searcher = searcher_from_config(rhodecode.CONFIG)
 
     search_result = searcher.search(
         "Test", document_type='commit', search_user=auth_user, repo_name=None)
     results = list(search_result['results'])
     assert len(results) == 1
     assert results[0]['repository'] == repo.repo_name
 
 
 def create_commit_index_with_one_document(search_location, repo_name):
     """
     Provides a test index based on our search schema.
 
     The full details of index creation are found inside of `rhodecode-tools`.
     The intention of this function is to provide just enough so that the
     search works.
     """
     test_index = index.create_in(
         search_location, whoosh.COMMIT_SCHEMA,
         indexname=whoosh.COMMIT_INDEX_NAME)
     writer = test_index.writer()
 
     writer.add_document(
         commit_id="fake_commit_id",
         commit_idx=1,
         owner="Test Owner",
         date=time.time(),
         repository=repo_name,
         author="Test Author",
         message="Test Message",
         added="added_1.txt added_2.txt",
         removed="removed_1.txt removed_2.txt",
         changed="changed_1.txt changed_2.txt",
         parents="fake_parent_1_id fake_parent_2_id",
         )
     writer.commit()