python3: 2to3 fixes

rhodecode/lib/_vendor/authomatic/core.py

@@ -1,1764 +1,1761 b''
 # -*- coding: utf-8 -*-
 
 import collections
 import copy
 import datetime
 import hashlib
 import hmac
 import json
 import logging
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
+import pickle
 import sys
 import threading
 import time
 from xml.etree import ElementTree
 
 from authomatic.exceptions import (
     ConfigError,
     CredentialsError,
     ImportStringError,
     RequestElementsError,
     SessionError,
 )
 from authomatic import six
 from authomatic.six.moves import urllib_parse as parse
 
 
 # =========================================================================
 # Global variables !!!
 # =========================================================================
 
 _logger = logging.getLogger(__name__)
 _logger.addHandler(logging.StreamHandler(sys.stdout))
 
 _counter = None
 
 
 def normalize_dict(dict_):
     """
     Replaces all values that are single-item iterables with the value of its
     index 0.
 
     :param dict dict_:
         Dictionary to normalize.
 
     :returns:
         Normalized dictionary.
 
     """
 
     return dict([(k, v[0] if not isinstance(v, str) and len(v) == 1 else v)
                  for k, v in list(dict_.items())])
 
 
 def items_to_dict(items):
     """
     Converts list of tuples to dictionary with duplicate keys converted to
     lists.
 
     :param list items:
         List of tuples.
 
     :returns:
         :class:`dict`
 
     """
 
     res = collections.defaultdict(list)
 
     for k, v in items:
         res[k].append(v)
 
     return normalize_dict(dict(res))
 
 
 class Counter(object):
     """
     A simple counter to be used in the config to generate unique `id` values.
     """
 
     def __init__(self, start=0):
         self._count = start
 
     def count(self):
         self._count += 1
         return self._count
 
 
 _counter = Counter()
 
 
 def provider_id():
     """
     A simple counter to be used in the config to generate unique `IDs`.
 
     :returns:
         :class:`int`.
 
     Use it in the :doc:`config` like this:
     ::
 
         import authomatic
 
         CONFIG = {
             'facebook': {
                  'class_': authomatic.providers.oauth2.Facebook,
                  'id': authomatic.provider_id(), # returns 1
                  'consumer_key': '##########',
                  'consumer_secret': '##########',
                  'scope': ['user_about_me', 'email']
             },
             'google': {
                  'class_': 'authomatic.providers.oauth2.Google',
                  'id': authomatic.provider_id(), # returns 2
                  'consumer_key': '##########',
                  'consumer_secret': '##########',
                  'scope': ['https://www.googleapis.com/auth/userinfo.profile',
                            'https://www.googleapis.com/auth/userinfo.email']
             },
             'windows_live': {
                  'class_': 'oauth2.WindowsLive',
                  'id': authomatic.provider_id(), # returns 3
                  'consumer_key': '##########',
                  'consumer_secret': '##########',
                  'scope': ['wl.basic', 'wl.emails', 'wl.photos']
             },
         }
 
     """
 
     return _counter.count()
 
 
 def escape(s):
     """
     Escape a URL including any /.
     """
     return parse.quote(s.encode('utf-8'), safe='~')
 
 
 def json_qs_parser(body):
     """
     Parses response body from JSON, XML or query string.
 
     :param body:
         string
 
     :returns:
         :class:`dict`, :class:`list` if input is JSON or query string,
         :class:`xml.etree.ElementTree.Element` if XML.
 
     """
     try:
         # Try JSON first.
         return json.loads(body)
     except (OverflowError, TypeError, ValueError):
         pass
 
     try:
         # Then XML.
         return ElementTree.fromstring(body)
     except (ElementTree.ParseError, TypeError, ValueError):
         pass
 
     # Finally query string.
     return dict(parse.parse_qsl(body))
 
 
 def import_string(import_name, silent=False):
     """
     Imports an object by string in dotted notation.
 
     taken `from webapp2.import_string() <http://webapp-
     improved.appspot.com/api/webapp2.html#webapp2.import_string>`_
 
     """
 
     try:
         if '.' in import_name:
             module, obj = import_name.rsplit('.', 1)
             return getattr(__import__(module, None, None, [obj]), obj)
         else:
             return __import__(import_name)
     except (ImportError, AttributeError) as e:
         if not silent:
             raise ImportStringError('Import from string failed for path {0}'
                                     .format(import_name), str(e))
 
 
 def resolve_provider_class(class_):
     """
     Returns a provider class.
 
     :param class_name: :class:`string` or
     :class:`authomatic.providers.BaseProvider` subclass.
 
     """
 
     if isinstance(class_, str):
         # prepare path for authomatic.providers package
         path = '.'.join([__package__, 'providers', class_])
 
         # try to import class by string from providers module or by fully
         # qualified path
         return import_string(class_, True) or import_string(path)
     else:
         return class_
 
 
 def id_to_name(config, short_name):
     """
     Returns the provider :doc:`config` key based on it's ``id`` value.
 
     :param dict config:
         :doc:`config`.
     :param id:
         Value of the id parameter in the :ref:`config` to search for.
 
     """
 
     for k, v in list(config.items()):
         if v.get('id') == short_name:
             return k
 
     raise Exception(
         'No provider with id={0} found in the config!'.format(short_name))
 
 
 class ReprMixin(object):
     """
     Provides __repr__() method with output *ClassName(arg1=value, arg2=value)*.
 
     Ignored are attributes
 
     * which values are considered false.
     * with leading underscore.
     * listed in _repr_ignore.
 
     Values of attributes listed in _repr_sensitive will be replaced by *###*.
     Values which repr() string is longer than _repr_length_limit will be
     represented as *ClassName(...)*
 
     """
 
     #: Iterable of attributes to be ignored.
     _repr_ignore = []
     #: Iterable of attributes which value should not be visible.
     _repr_sensitive = []
     #: `int` Values longer than this will be truncated to *ClassName(...)*.
     _repr_length_limit = 20
 
     def __repr__(self):
 
         # get class name
         name = self.__class__.__name__
 
         # construct keyword arguments
         args = []
 
         for k, v in list(self.__dict__.items()):
 
             # ignore attributes with leading underscores and those listed in
             # _repr_ignore
             if v and not k.startswith('_') and k not in self._repr_ignore:
 
                 # replace sensitive values
                 if k in self._repr_sensitive:
                     v = '###'
 
                 # if repr is too long
                 if len(repr(v)) > self._repr_length_limit:
                     # Truncate to ClassName(...)
                     v = '{0}(...)'.format(v.__class__.__name__)
                 else:
                     v = repr(v)
 
                 args.append('{0}={1}'.format(k, v))
 
         return '{0}({1})'.format(name, ', '.join(args))
 
 
 class Future(threading.Thread):
     """
     Represents an activity run in a separate thread. Subclasses the standard
     library :class:`threading.Thread` and adds :attr:`.get_result` method.
 
     .. warning::
 
         |async|
 
     """
 
     def __init__(self, func, *args, **kwargs):
         """
         :param callable func:
             The function to be run in separate thread.
 
         Calls :data:`func` in separate thread and returns immediately.
         Accepts arbitrary positional and keyword arguments which will be
         passed to :data:`func`.
         """
 
         super(Future, self).__init__()
         self._func = func
         self._args = args
         self._kwargs = kwargs
         self._result = None
 
         self.start()
 
     def run(self):
         self._result = self._func(*self._args, **self._kwargs)
 
     def get_result(self, timeout=None):
         """
         Waits for the wrapped :data:`func` to finish and returns its result.
 
         .. note::
 
             This will block the **calling thread** until the :data:`func`
             returns.
 
         :param timeout:
             :class:`float` or ``None`` A timeout for the :data:`func` to
             return in seconds.
 
         :returns:
             The result of the wrapped :data:`func`.
 
         """
 
         self.join(timeout)
         return self._result
 
 
 class Session(object):
     """
     A dictionary-like secure cookie session implementation.
     """
 
     def __init__(self, adapter, secret, name='authomatic', max_age=600,
                  secure=False):
         """
         :param str secret:
             Session secret used to sign the session cookie.
         :param str name:
             Session cookie name.
         :param int max_age:
             Maximum allowed age of session cookie nonce in seconds.
         :param bool secure:
             If ``True`` the session cookie will be saved with ``Secure``
             attribute.
         """
 
         self.adapter = adapter
         self.name = name
         self.secret = secret
         self.max_age = max_age
         self.secure = secure
         self._data = {}
 
     def create_cookie(self, delete=None):
         """
         Creates the value for ``Set-Cookie`` HTTP header.
 
         :param bool delete:
             If ``True`` the cookie value will be ``deleted`` and the
             Expires value will be ``Thu, 01-Jan-1970 00:00:01 GMT``.
 
         """
         value = 'deleted' if delete else self._serialize(self.data)
         split_url = parse.urlsplit(self.adapter.url)
         domain = split_url.netloc.split(':')[0]
 
         # Work-around for issue #11, failure of WebKit-based browsers to accept
         # cookies set as part of a redirect response in some circumstances.
         if '.' not in domain:
             template = '{name}={value}; Path={path}; HttpOnly{secure}{expires}'
         else:
             template = ('{name}={value}; Domain={domain}; Path={path}; '
                         'HttpOnly{secure}{expires}')
 
         return template.format(
             name=self.name,
             value=value,
             domain=domain,
             path=split_url.path,
             secure='; Secure' if self.secure else '',
             expires='; Expires=Thu, 01-Jan-1970 00:00:01 GMT' if delete else ''
         )
 
     def save(self):
         """
         Adds the session cookie to headers.
         """
         if self.data:
             cookie = self.create_cookie()
             cookie_len = len(cookie)
 
             if cookie_len > 4093:
                 raise SessionError('Cookie too long! The cookie size {0} '
                                    'is more than 4093 bytes.'
                                    .format(cookie_len))
 
             self.adapter.set_header('Set-Cookie', cookie)
 
             # Reset data
             self._data = {}
 
     def delete(self):
         self.adapter.set_header('Set-Cookie', self.create_cookie(delete=True))
 
     def _get_data(self):
         """
         Extracts the session data from cookie.
         """
         cookie = self.adapter.cookies.get(self.name)
         return self._deserialize(cookie) if cookie else {}
 
     @property
     def data(self):
         """
         Gets session data lazily.
         """
         if not self._data:
             self._data = self._get_data()
         # Always return a dict, even if deserialization returned nothing
         if self._data is None:
             self._data = {}
         return self._data
 
     def _signature(self, *parts):
         """
         Creates signature for the session.
         """
         signature = hmac.new(six.b(self.secret), digestmod=hashlib.sha1)
         signature.update(six.b('|'.join(parts)))
         return signature.hexdigest()
 
     def _serialize(self, value):
         """
         Converts the value to a signed string with timestamp.
 
         :param value:
             Object to be serialized.
 
         :returns:
             Serialized value.
 
         """
 
         # data = copy.deepcopy(value)
         data = value
 
         # 1. Serialize
         serialized = pickle.dumps(data).decode('latin-1')
 
         # 2. Encode
         # Percent encoding produces smaller result then urlsafe base64.
         encoded = parse.quote(serialized, '')
 
         # 3. Concatenate
         timestamp = str(int(time.time()))
         signature = self._signature(self.name, encoded, timestamp)
         concatenated = '|'.join([encoded, timestamp, signature])
 
         return concatenated
 
     def _deserialize(self, value):
         """
         Deserializes and verifies the value created by :meth:`._serialize`.
 
         :param str value:
             The serialized value.
 
         :returns:
             Deserialized object.
 
         """
 
         # 3. Split
         encoded, timestamp, signature = value.split('|')
 
         # Verify signature
         if not signature == self._signature(self.name, encoded, timestamp):
             raise SessionError('Invalid signature "{0}"!'.format(signature))
 
         # Verify timestamp
         if int(timestamp) < int(time.time()) - self.max_age:
             return None
 
         # 2. Decode
         decoded = parse.unquote(encoded)
 
         # 1. Deserialize
         deserialized = pickle.loads(decoded.encode('latin-1'))
 
         return deserialized
 
     def __setitem__(self, key, value):
         self._data[key] = value
 
     def __getitem__(self, key):
         return self.data.__getitem__(key)
 
     def __delitem__(self, key):
         return self._data.__delitem__(key)
 
     def get(self, key, default=None):
         return self.data.get(key, default)
 
 
 class User(ReprMixin):
     """
     Provides unified interface to selected **user** info returned by different
     **providers**.
 
     .. note:: The value format may vary across providers.
 
     """
 
     def __init__(self, provider, **kwargs):
         #: A :doc:`provider <providers>` instance.
         self.provider = provider
 
         #: An :class:`.Credentials` instance.
         self.credentials = kwargs.get('credentials')
 
         #: A :class:`dict` containing all the **user** information returned
         #: by the **provider**.
         #: The structure differs across **providers**.
         self.data = kwargs.get('data')
 
         #: The :attr:`.Response.content` of the request made to update
         #: the user.
         self.content = kwargs.get('content')
 
         #: :class:`str` ID assigned to the **user** by the **provider**.
         self.id = kwargs.get('id')
         #: :class:`str` User name e.g. *andrewpipkin*.
         self.username = kwargs.get('username')
         #: :class:`str` Name e.g. *Andrew Pipkin*.
         self.name = kwargs.get('name')
         #: :class:`str` First name e.g. *Andrew*.
         self.first_name = kwargs.get('first_name')
         #: :class:`str` Last name e.g. *Pipkin*.
         self.last_name = kwargs.get('last_name')
         #: :class:`str` Nickname e.g. *Andy*.
         self.nickname = kwargs.get('nickname')
         #: :class:`str` Link URL.
         self.link = kwargs.get('link')
         #: :class:`str` Gender.
         self.gender = kwargs.get('gender')
         #: :class:`str` Timezone.
         self.timezone = kwargs.get('timezone')
         #: :class:`str` Locale.
         self.locale = kwargs.get('locale')
         #: :class:`str` E-mail.
         self.email = kwargs.get('email')
         #: :class:`str` phone.
         self.phone = kwargs.get('phone')
         #: :class:`str` Picture URL.
         self.picture = kwargs.get('picture')
         #: Birth date as :class:`datetime.datetime()` or :class:`str`
         #  if parsing failed or ``None``.
         self.birth_date = kwargs.get('birth_date')
         #: :class:`str` Country.
         self.country = kwargs.get('country')
         #: :class:`str` City.
         self.city = kwargs.get('city')
         #: :class:`str` Geographical location.
         self.location = kwargs.get('location')
         #: :class:`str` Postal code.
         self.postal_code = kwargs.get('postal_code')
         #: Instance of the Google App Engine Users API
         #: `User <https://developers.google.com/appengine/docs/python/users/userclass>`_ class.
         #: Only present when using the :class:`authomatic.providers.gaeopenid.GAEOpenID` provider.
         self.gae_user = kwargs.get('gae_user')
 
     def update(self):
         """
         Updates the user info by fetching the **provider's** user info URL.
 
         :returns:
             Updated instance of this class.
 
         """
 
         return self.provider.update_user()
 
     def async_update(self):
         """
         Same as :meth:`.update` but runs asynchronously in a separate thread.
 
         .. warning::
 
             |async|
 
         :returns:
             :class:`.Future` instance representing the separate thread.
 
         """
 
         return Future(self.update)
 
     def to_dict(self):
         """
         Converts the :class:`.User` instance to a :class:`dict`.
 
         :returns:
             :class:`dict`
 
         """
 
         # copy the dictionary
         d = copy.copy(self.__dict__)
 
         # Keep only the provider name to avoid circular reference
         d['provider'] = self.provider.name
         d['credentials'] = self.credentials.serialize(
         ) if self.credentials else None
         d['birth_date'] = str(d['birth_date'])
 
         # Remove content
         d.pop('content')
 
         if isinstance(self.data, ElementTree.Element):
             d['data'] = None
 
         return d
 
 
 SupportedUserAttributesNT = collections.namedtuple(
     typename='SupportedUserAttributesNT',
     field_names=['birth_date', 'city', 'country', 'email', 'first_name',
                  'gender', 'id', 'last_name', 'link', 'locale', 'location',
                  'name', 'nickname', 'phone', 'picture', 'postal_code',
                  'timezone', 'username', ]
 )
 
 
 class SupportedUserAttributes(SupportedUserAttributesNT):
     def __new__(cls, **kwargs):
         defaults = dict((i, False) for i in SupportedUserAttributes._fields)  # pylint:disable=no-member
         defaults.update(**kwargs)
         return super(SupportedUserAttributes, cls).__new__(cls, **defaults)
 
 
 class Credentials(ReprMixin):
     """
     Contains all necessary information to fetch **user's protected resources**.
     """
 
     _repr_sensitive = ('token', 'refresh_token', 'token_secret',
                        'consumer_key', 'consumer_secret')
 
     def __init__(self, config, **kwargs):
 
         #: :class:`dict` :doc:`config`.
         self.config = config
 
         #: :class:`str` User **access token**.
         self.token = kwargs.get('token', '')
 
         #: :class:`str` Access token type.
         self.token_type = kwargs.get('token_type', '')
 
         #: :class:`str` Refresh token.
         self.refresh_token = kwargs.get('refresh_token', '')
 
         #: :class:`str` Access token secret.
         self.token_secret = kwargs.get('token_secret', '')
 
         #: :class:`int` Expiration date as UNIX timestamp.
         self.expiration_time = int(kwargs.get('expiration_time', 0))
 
         #: A :doc:`Provider <providers>` instance**.
         provider = kwargs.get('provider')
 
         self.expire_in = int(kwargs.get('expire_in', 0))
 
         if provider:
             #: :class:`str` Provider name specified in the :doc:`config`.
             self.provider_name = provider.name
 
             #: :class:`str` Provider type e.g.
             #  ``"authomatic.providers.oauth2.OAuth2"``.
             self.provider_type = provider.get_type()
 
             #: :class:`str` Provider type e.g.
             #  ``"authomatic.providers.oauth2.OAuth2"``.
             self.provider_type_id = provider.type_id
 
             #: :class:`str` Provider short name specified in the :doc:`config`.
             self.provider_id = int(provider.id) if provider.id else None
 
             #: :class:`class` Provider class.
             self.provider_class = provider.__class__
 
             #: :class:`str` Consumer key specified in the :doc:`config`.
             self.consumer_key = provider.consumer_key
 
             #: :class:`str` Consumer secret specified in the :doc:`config`.
             self.consumer_secret = provider.consumer_secret
 
         else:
             self.provider_name = kwargs.get('provider_name', '')
             self.provider_type = kwargs.get('provider_type', '')
             self.provider_type_id = kwargs.get('provider_type_id')
             self.provider_id = kwargs.get('provider_id')
             self.provider_class = kwargs.get('provider_class')
 
             self.consumer_key = kwargs.get('consumer_key', '')
             self.consumer_secret = kwargs.get('consumer_secret', '')
 
     @property
     def expire_in(self):
         """
 
         """
 
         return self._expire_in
 
     @expire_in.setter
     def expire_in(self, value):
         """
         Computes :attr:`.expiration_time` when the value is set.
         """
 
         # pylint:disable=attribute-defined-outside-init
         if value:
             self._expiration_time = int(time.time()) + int(value)
             self._expire_in = value
 
     @property
     def expiration_time(self):
         return self._expiration_time
 
     @expiration_time.setter
     def expiration_time(self, value):
 
         # pylint:disable=attribute-defined-outside-init
         self._expiration_time = int(value)
         self._expire_in = self._expiration_time - int(time.time())
 
     @property
     def expiration_date(self):
         """
         Expiration date as :class:`datetime.datetime` or ``None`` if
         credentials never expire.
         """
 
         if self.expire_in < 0:
             return None
         else:
             return datetime.datetime.fromtimestamp(self.expiration_time)
 
     @property
     def valid(self):
         """
         ``True`` if credentials are valid, ``False`` if expired.
         """
 
         if self.expiration_time:
             return self.expiration_time > int(time.time())
         else:
             return True
 
     def expire_soon(self, seconds):
         """
         Returns ``True`` if credentials expire sooner than specified.
 
         :param int seconds:
             Number of seconds.
 
         :returns:
             ``True`` if credentials expire sooner than specified,
             else ``False``.
 
         """
 
         if self.expiration_time:
             return self.expiration_time < int(time.time()) + int(seconds)
         else:
             return False
 
     def refresh(self, force=False, soon=86400):
         """
         Refreshes the credentials only if the **provider** supports it and if
         it will expire in less than one day. It does nothing in other cases.
 
         .. note::
 
             The credentials will be refreshed only if it gives sense
             i.e. only |oauth2|_ has the notion of credentials
             *refreshment/extension*.
             And there are also differences across providers e.g. Google
             supports refreshment only if there is a ``refresh_token`` in
             the credentials and that in turn is present only if the
             ``access_type`` parameter was set to ``offline`` in the
             **user authorization request**.
 
         :param bool force:
             If ``True`` the credentials will be refreshed even if they
             won't expire soon.
 
         :param int soon:
             Number of seconds specifying what means *soon*.
 
         """
 
         if hasattr(self.provider_class, 'refresh_credentials'):
             if force or self.expire_soon(soon):
                 logging.info('PROVIDER NAME: {0}'.format(self.provider_name))
                 return self.provider_class(
                     self, None, self.provider_name).refresh_credentials(self)
 
     def async_refresh(self, *args, **kwargs):
         """
         Same as :meth:`.refresh` but runs asynchronously in a separate thread.
 
         .. warning::
 
             |async|
 
         :returns:
             :class:`.Future` instance representing the separate thread.
 
         """
 
         return Future(self.refresh, *args, **kwargs)
 
     def provider_type_class(self):
         """
         Returns the :doc:`provider <providers>` class specified in the
         :doc:`config`.
 
         :returns:
             :class:`authomatic.providers.BaseProvider` subclass.
 
         """
 
         return resolve_provider_class(self.provider_type)
 
     def serialize(self):
         """
         Converts the credentials to a percent encoded string to be stored for
         later use.
 
         :returns:
             :class:`string`
 
         """
 
         if self.provider_id is None:
             raise ConfigError(
                 'To serialize credentials you need to specify a '
                 'unique integer under the "id" key in the config '
                 'for each provider!')
 
         # Get the provider type specific items.
         rest = self.provider_type_class().to_tuple(self)
 
         # Provider ID and provider type ID are always the first two items.
         result = (self.provider_id, self.provider_type_id) + rest
 
         # Make sure that all items are strings.
         stringified = [str(i) for i in result]
 
         # Concatenate by newline.
         concatenated = '\n'.join(stringified)
 
         # Percent encode.
         return parse.quote(concatenated, '')
 
     @classmethod
     def deserialize(cls, config, credentials):
         """
         A *class method* which reconstructs credentials created by
         :meth:`serialize`. You can also pass it a :class:`.Credentials`
         instance.
 
         :param dict config:
             The same :doc:`config` used in the :func:`.login` to get the
             credentials.
         :param str credentials:
             :class:`string` The serialized credentials or
             :class:`.Credentials` instance.
 
         :returns:
             :class:`.Credentials`
 
         """
 
         # Accept both serialized and normal.
         if isinstance(credentials, Credentials):
             return credentials
 
         decoded = parse.unquote(credentials)
 
         split = decoded.split('\n')
 
         # We need the provider ID to move forward.
         if split[0] is None:
             raise CredentialsError(
                 'To deserialize credentials you need to specify a unique '
                 'integer under the "id" key in the config for each provider!')
 
         # Get provider config by short name.
         provider_name = id_to_name(config, int(split[0]))
         cfg = config.get(provider_name)
 
         # Get the provider class.
         ProviderClass = resolve_provider_class(cfg.get('class_'))
 
         deserialized = Credentials(config)
 
         deserialized.provider_id = provider_id
         deserialized.provider_type = ProviderClass.get_type()
         deserialized.provider_type_id = split[1]
         deserialized.provider_class = ProviderClass
         deserialized.provider_name = provider_name
         deserialized.provider_class = ProviderClass
 
         # Add provider type specific properties.
         return ProviderClass.reconstruct(split[2:], deserialized, cfg)
 
 
 class LoginResult(ReprMixin):
     """
     Result of the :func:`authomatic.login` function.
     """
 
     def __init__(self, provider):
         #: A :doc:`provider <providers>` instance.
         self.provider = provider
 
         #: An instance of the :exc:`authomatic.exceptions.BaseError` subclass.
         self.error = None
 
     def popup_js(self, callback_name=None, indent=None,
                  custom=None, stay_open=False):
         """
         Returns JavaScript that:
 
         #.  Triggers the ``options.onLoginComplete(result, closer)``
             handler set with the :ref:`authomatic.setup() <js_setup>`
             function of :ref:`javascript.js <js>`.
         #.  Calls the JavasScript callback specified by :data:`callback_name`
             on the opener of the *login handler popup* and passes it the
             *login result* JSON object as first argument and the `closer`
             function which you should call in your callback to close the popup.
 
         :param str callback_name:
             The name of the javascript callback e.g ``foo.bar.loginCallback``
             will result in ``window.opener.foo.bar.loginCallback(result);``
             in the HTML.
 
         :param int indent:
             The number of spaces to indent the JSON result object.
             If ``0`` or negative, only newlines are added.
             If ``None``, no newlines are added.
 
         :param custom:
             Any JSON serializable object that will be passed to the
             ``result.custom`` attribute.
 
         :param str stay_open:
             If ``True``, the popup will stay open.
 
         :returns:
             :class:`str` with JavaScript.
 
         """
 
         custom_callback = """
         try {{ window.opener.{cb}(result, closer); }} catch(e) {{}}
         """.format(cb=callback_name) if callback_name else ''
 
         # TODO: Move the window.close() to the opener
         return """
         (function(){{
 
             closer = function(){{
                 window.close();
             }};
 
             var result = {result};
             result.custom = {custom};
 
             {custom_callback}
 
             try {{
                 window.opener.authomatic.loginComplete(result, closer);
             }} catch(e) {{}}
 
         }})();
 
         """.format(result=self.to_json(indent),
                    custom=json.dumps(custom),
                    custom_callback=custom_callback,
                    stay_open='// ' if stay_open else '')
 
     def popup_html(self, callback_name=None, indent=None,
                    title='Login | {0}', custom=None, stay_open=False):
         """
         Returns a HTML with JavaScript that:
 
         #.  Triggers the ``options.onLoginComplete(result, closer)`` handler
             set with the :ref:`authomatic.setup() <js_setup>` function of
             :ref:`javascript.js <js>`.
         #.  Calls the JavasScript callback specified by :data:`callback_name`
             on the opener of the *login handler popup* and passes it the
             *login result* JSON object as first argument and the `closer`
             function which you should call in your callback to close the popup.
 
         :param str callback_name:
             The name of the javascript callback e.g ``foo.bar.loginCallback``
             will result in ``window.opener.foo.bar.loginCallback(result);``
             in the HTML.
 
         :param int indent:
             The number of spaces to indent the JSON result object.
             If ``0`` or negative, only newlines are added.
             If ``None``, no newlines are added.
 
         :param str title:
             The text of the HTML title. You can use ``{0}`` tag inside,
             which will be replaced by the provider name.
 
         :param custom:
             Any JSON serializable object that will be passed to the
             ``result.custom`` attribute.
 
         :param str stay_open:
             If ``True``, the popup will stay open.
 
         :returns:
             :class:`str` with HTML.
 
         """
 
         return """
         <!DOCTYPE html>
         <html>
             <head><title>{title}</title></head>
             <body>
             <script type="text/javascript">
                 {js}
             </script>
             </body>
         </html>
         """.format(
             title=title.format(self.provider.name if self.provider else ''),
             js=self.popup_js(callback_name, indent, custom, stay_open)
         )
 
     @property
     def user(self):
         """
         A :class:`.User` instance.
         """
 
         return self.provider.user if self.provider else None
 
     def to_dict(self):
         return dict(provider=self.provider, user=self.user, error=self.error)
 
     def to_json(self, indent=4):
         return json.dumps(self, default=lambda obj: obj.to_dict(
         ) if hasattr(obj, 'to_dict') else '', indent=indent)
 
 
 class Response(ReprMixin):
     """
     Wraps :class:`httplib.HTTPResponse` and adds.
 
     :attr:`.content` and :attr:`.data` attributes.
 
     """
 
     def __init__(self, httplib_response, content_parser=None):
         """
         :param httplib_response:
             The wrapped :class:`httplib.HTTPResponse` instance.
 
         :param function content_parser:
             Callable which accepts :attr:`.content` as argument,
             parses it and returns the parsed data as :class:`dict`.
         """
 
         self.httplib_response = httplib_response
         self.content_parser = content_parser or json_qs_parser
         self._data = None
         self._content = None
 
         #: Same as :attr:`httplib.HTTPResponse.msg`.
         self.msg = httplib_response.msg
         #: Same as :attr:`httplib.HTTPResponse.version`.
         self.version = httplib_response.version
         #: Same as :attr:`httplib.HTTPResponse.status`.
         self.status = httplib_response.status
         #: Same as :attr:`httplib.HTTPResponse.reason`.
         self.reason = httplib_response.reason
 
     def read(self, amt=None):
         """
         Same as :meth:`httplib.HTTPResponse.read`.
 
         :param amt:
 
         """
 
         return self.httplib_response.read(amt)
 
     def getheader(self, name, default=None):
         """
         Same as :meth:`httplib.HTTPResponse.getheader`.
 
         :param name:
         :param default:
 
         """
 
         return self.httplib_response.getheader(name, default)
 
     def fileno(self):
         """
         Same as :meth:`httplib.HTTPResponse.fileno`.
         """
         return self.httplib_response.fileno()
 
     def getheaders(self):
         """
         Same as :meth:`httplib.HTTPResponse.getheaders`.
         """
         return self.httplib_response.getheaders()
 
     @staticmethod
     def is_binary_string(content):
         """
         Return true if string is binary data.
         """
 
         textchars = (bytearray([7, 8, 9, 10, 12, 13, 27]) +
                      bytearray(range(0x20, 0x100)))
         return bool(content.translate(None, textchars))
 
     @property
     def content(self):
         """
         The whole response content.
         """
 
         if not self._content:
             content = self.httplib_response.read()
             if self.is_binary_string(content):
                 self._content = content
             else:
                 self._content = content.decode('utf-8')
         return self._content
 
     @property
     def data(self):
         """
         A :class:`dict` of data parsed from :attr:`.content`.
         """
 
         if not self._data:
             self._data = self.content_parser(self.content)
         return self._data
 
 
 class UserInfoResponse(Response):
     """
     Inherits from :class:`.Response`, adds  :attr:`~UserInfoResponse.user`
     attribute.
     """
 
     def __init__(self, user, *args, **kwargs):
         super(UserInfoResponse, self).__init__(*args, **kwargs)
 
         #: :class:`.User` instance.
         self.user = user
 
 
 class RequestElements(tuple):
     """
     A tuple of ``(url, method, params, headers, body)`` request elements.
 
     With some additional properties.
 
     """
 
     def __new__(cls, url, method, params, headers, body):
         return tuple.__new__(cls, (url, method, params, headers, body))
 
     @property
     def url(self):
         """
         Request URL.
         """
 
         return self[0]
 
     @property
     def method(self):
         """
         HTTP method of the request.
         """
 
         return self[1]
 
     @property
     def params(self):
         """
         Dictionary of request parameters.
         """
 
         return self[2]
 
     @property
     def headers(self):
         """
         Dictionary of request headers.
         """
 
         return self[3]
 
     @property
     def body(self):
         """
         :class:`str` Body of ``POST``, ``PUT`` and ``PATCH`` requests.
         """
 
         return self[4]
 
     @property
     def query_string(self):
         """
         Query string of the request.
         """
 
         return parse.urlencode(self.params)
 
     @property
     def full_url(self):
         """
         URL with query string.
         """
 
         return self.url + '?' + self.query_string
 
     def to_json(self):
         return json.dumps(dict(url=self.url,
                                method=self.method,
                                params=self.params,
                                headers=self.headers,
                                body=self.body))
 
 
 class Authomatic(object):
     def __init__(
             self, config, secret, session_max_age=600, secure_cookie=False,
             session=None, session_save_method=None, report_errors=True,
             debug=False, logging_level=logging.INFO, prefix='authomatic',
             logger=None
     ):
         """
         Encapsulates all the functionality of this package.
 
         :param dict config:
             :doc:`config`
 
         :param str secret:
             A secret string that will be used as the key for signing
             :class:`.Session` cookie and as a salt by *CSRF* token generation.
 
         :param session_max_age:
             Maximum allowed age of :class:`.Session` cookie nonce in seconds.
 
         :param bool secure_cookie:
             If ``True`` the :class:`.Session` cookie will be saved wit
             ``Secure`` attribute.
 
         :param session:
             Custom dictionary-like session implementation.
 
         :param callable session_save_method:
             A method of the supplied session or any mechanism that saves the
             session data and cookie.
 
         :param bool report_errors:
             If ``True`` exceptions encountered during the **login procedure**
             will be caught and reported in the :attr:`.LoginResult.error`
             attribute.
             Default is ``True``.
 
         :param bool debug:
             If ``True`` traceback of exceptions will be written to response.
             Default is ``False``.
 
         :param int logging_level:
             The logging level threshold for the default logger as specified in
             the standard Python
             `logging library <http://docs.python.org/2/library/logging.html>`_.
             This setting is ignored when :data:`logger` is set.
             Default is ``logging.INFO``.
 
         :param str prefix:
             Prefix used as the :class:`.Session` cookie name.
 
         :param logger:
             A :class:`logging.logger` instance.
 
         """
 
         self.config = config
         self.secret = secret
         self.session_max_age = session_max_age
         self.secure_cookie = secure_cookie
         self.session = session
         self.session_save_method = session_save_method
         self.report_errors = report_errors
         self.debug = debug
         self.logging_level = logging_level
         self.prefix = prefix
         self._logger = logger or logging.getLogger(str(id(self)))
 
         # Set logging level.
         if logger is None:
             self._logger.setLevel(logging_level)
 
     def login(self, adapter, provider_name, callback=None,
               session=None, session_saver=None, **kwargs):
         """
         If :data:`provider_name` specified, launches the login procedure for
         corresponding :doc:`provider </reference/providers>` and returns
         :class:`.LoginResult`.
 
         If :data:`provider_name` is empty, acts like
         :meth:`.Authomatic.backend`.
 
         .. warning::
 
             The method redirects the **user** to the **provider** which in
             turn redirects **him/her** back to the *request handler* where
             it has been called.
 
         :param str provider_name:
             Name of the provider as specified in the keys of the :doc:`config`.
 
         :param callable callback:
             If specified the method will call the callback with
             :class:`.LoginResult` passed as argument and will return nothing.
 
         :param bool report_errors:
 
         .. note::
 
             Accepts additional keyword arguments that will be passed to
             :doc:`provider <providers>` constructor.
 
         :returns:
             :class:`.LoginResult`
 
         """
 
         if provider_name:
             # retrieve required settings for current provider and raise
             # exceptions if missing
             provider_settings = self.config.get(provider_name)
             if not provider_settings:
                 raise ConfigError('Provider name "{0}" not specified!'
                                   .format(provider_name))
 
             if not (session is None or session_saver is None):
                 session = session
                 session_saver = session_saver
             else:
                 session = Session(adapter=adapter,
                                   secret=self.secret,
                                   max_age=self.session_max_age,
                                   name=self.prefix,
                                   secure=self.secure_cookie)
 
                 session_saver = session.save
 
             # Resolve provider class.
             class_ = provider_settings.get('class_')
             if not class_:
                 raise ConfigError(
                     'The "class_" key not specified in the config'
                     ' for provider {0}!'.format(provider_name))
             ProviderClass = resolve_provider_class(class_)
 
             # FIXME: Find a nicer solution
             ProviderClass._logger = self._logger
 
             # instantiate provider class
             provider = ProviderClass(self,
                                      adapter=adapter,
                                      provider_name=provider_name,
                                      callback=callback,
                                      session=session,
                                      session_saver=session_saver,
                                      **kwargs)
 
             # return login result
             return provider.login()
 
         else:
             # Act like backend.
             self.backend(adapter)
 
     def credentials(self, credentials):
         """
         Deserializes credentials.
 
         :param credentials:
             Credentials serialized with :meth:`.Credentials.serialize` or
             :class:`.Credentials` instance.
 
         :returns:
             :class:`.Credentials`
 
         """
 
         return Credentials.deserialize(self.config, credentials)
 
     def access(self, credentials, url, params=None, method='GET',
                headers=None, body='', max_redirects=5, content_parser=None):
         """
         Accesses **protected resource** on behalf of the **user**.
 
         :param credentials:
             The **user's** :class:`.Credentials` (serialized or normal).
 
         :param str url:
             The **protected resource** URL.
 
         :param str method:
             HTTP method of the request.
 
         :param dict headers:
             HTTP headers of the request.
 
         :param str body:
             Body of ``POST``, ``PUT`` and ``PATCH`` requests.
 
         :param int max_redirects:
             Maximum number of HTTP redirects to follow.
 
         :param function content_parser:
             A function to be used to parse the :attr:`.Response.data`
             from :attr:`.Response.content`.
 
         :returns:
             :class:`.Response`
 
         """
 
         # Deserialize credentials.
         credentials = Credentials.deserialize(self.config, credentials)
 
         # Resolve provider class.
         ProviderClass = credentials.provider_class
         logging.info('ACCESS HEADERS: {0}'.format(headers))
         # Access resource and return response.
 
         provider = ProviderClass(
             self, adapter=None, provider_name=credentials.provider_name)
         provider.credentials = credentials
 
         return provider.access(url=url,
                                params=params,
                                method=method,
                                headers=headers,
                                body=body,
                                max_redirects=max_redirects,
                                content_parser=content_parser)
 
     def async_access(self, *args, **kwargs):
         """
         Same as :meth:`.Authomatic.access` but runs asynchronously in a
         separate thread.
 
         .. warning::
 
             |async|
 
         :returns:
             :class:`.Future` instance representing the separate thread.
 
         """
 
         return Future(self.access, *args, **kwargs)
 
     def request_elements(
             self, credentials=None, url=None, method='GET', params=None,
             headers=None, body='', json_input=None, return_json=False
     ):
         """
         Creates request elements for accessing **protected resource of a
         user**. Required arguments are :data:`credentials` and :data:`url`. You
         can pass :data:`credentials`, :data:`url`, :data:`method`, and
         :data:`params` as a JSON object.
 
         :param credentials:
             The **user's** credentials (can be serialized).
 
         :param str url:
             The url of the protected resource.
 
         :param str method:
             The HTTP method of the request.
 
         :param dict params:
             Dictionary of request parameters.
 
         :param dict headers:
             Dictionary of request headers.
 
         :param str body:
             Body of ``POST``, ``PUT`` and ``PATCH`` requests.
 
         :param str json_input:
             you can pass :data:`credentials`, :data:`url`, :data:`method`,
             :data:`params` and :data:`headers` in a JSON object.
             Values from arguments will be used for missing properties.
 
             ::
 
                 {
                     "credentials": "###",
                     "url": "https://example.com/api",
                     "method": "POST",
                     "params": {
                         "foo": "bar"
                     },
                     "headers": {
                         "baz": "bing",
                         "Authorization": "Bearer ###"
                     },
                     "body": "Foo bar baz bing."
                 }
 
         :param bool return_json:
             if ``True`` the function returns a json object.
 
             ::
 
                 {
                     "url": "https://example.com/api",
                     "method": "POST",
                     "params": {
                         "access_token": "###",
                         "foo": "bar"
                     },
                     "headers": {
                         "baz": "bing",
                         "Authorization": "Bearer ###"
                     },
                     "body": "Foo bar baz bing."
                 }
 
         :returns:
             :class:`.RequestElements` or JSON string.
 
         """
 
         # Parse values from JSON
         if json_input:
             parsed_input = json.loads(json_input)
 
             credentials = parsed_input.get('credentials', credentials)
             url = parsed_input.get('url', url)
             method = parsed_input.get('method', method)
             params = parsed_input.get('params', params)
             headers = parsed_input.get('headers', headers)
             body = parsed_input.get('body', body)
 
         if not credentials and url:
             raise RequestElementsError(
                 'To create request elements, you must provide credentials '
                 'and URL either as keyword arguments or in the JSON object!')
 
         # Get the provider class
         credentials = Credentials.deserialize(self.config, credentials)
         ProviderClass = credentials.provider_class
 
         # Create request elements
         request_elements = ProviderClass.create_request_elements(
             ProviderClass.PROTECTED_RESOURCE_REQUEST_TYPE,
             credentials=credentials,
             url=url,
             method=method,
             params=params,
             headers=headers,
             body=body)
 
         if return_json:
             return request_elements.to_json()
 
         else:
             return request_elements
 
     def backend(self, adapter):
         """
         Converts a *request handler* to a JSON backend which you can use with
         :ref:`authomatic.js <js>`.
 
         Just call it inside a *request handler* like this:
 
         ::
 
             class JSONHandler(webapp2.RequestHandler):
                 def get(self):
                     authomatic.backend(Webapp2Adapter(self))
 
         :param adapter:
             The only argument is an :doc:`adapter <adapters>`.
 
         The *request handler* will now accept these request parameters:
 
         :param str type:
             Type of the request. Either ``auto``, ``fetch`` or ``elements``.
             Default is ``auto``.
 
         :param str credentials:
             Serialized :class:`.Credentials`.
 
         :param str url:
             URL of the **protected resource** request.
 
         :param str method:
             HTTP method of the **protected resource** request.
 
         :param str body:
             HTTP body of the **protected resource** request.
 
         :param JSON params:
             HTTP params of the **protected resource** request as a JSON object.
 
         :param JSON headers:
             HTTP headers of the **protected resource** request as a
             JSON object.
 
         :param JSON json:
             You can pass all of the aforementioned params except ``type``
             in a JSON object.
 
             .. code-block:: javascript
 
                 {
                     "credentials": "######",
                     "url": "https://example.com",
                     "method": "POST",
                     "params": {"foo": "bar"},
                     "headers": {"baz": "bing"},
                     "body": "the body of the request"
                 }
 
         Depending on the ``type`` param, the handler will either write
         a JSON object with *request elements* to the response,
         and add an ``Authomatic-Response-To: elements`` response header, ...
 
         .. code-block:: javascript
 
             {
                 "url": "https://example.com/api",
                 "method": "POST",
                 "params": {
                     "access_token": "###",
                     "foo": "bar"
                 },
                 "headers": {
                     "baz": "bing",
                     "Authorization": "Bearer ###"
                 }
             }
 
         ... or make a fetch to the **protected resource** and forward
         it's response content, status and headers with an additional
         ``Authomatic-Response-To: fetch`` header to the response.
 
         .. warning::
 
             The backend will not work if you write anything to the
             response in the handler!
 
         """
 
         AUTHOMATIC_HEADER = 'Authomatic-Response-To'
 
         # Collect request params
         request_type = adapter.params.get('type', 'auto')
         json_input = adapter.params.get('json')
         credentials = adapter.params.get('credentials')
         url = adapter.params.get('url')
         method = adapter.params.get('method', 'GET')
         body = adapter.params.get('body', '')
 
         params = adapter.params.get('params')
         params = json.loads(params) if params else {}
 
         headers = adapter.params.get('headers')
         headers = json.loads(headers) if headers else {}
 
         ProviderClass = Credentials.deserialize(
             self.config, credentials).provider_class
 
         if request_type == 'auto':
             # If there is a "callback" param, it's a JSONP request.
             jsonp = params.get('callback')
 
             # JSONP is possible only with GET method.
             if ProviderClass.supports_jsonp and method is 'GET':
                 request_type = 'elements'
             else:
                 # Remove the JSONP callback
                 if jsonp:
                     params.pop('callback')
                 request_type = 'fetch'
 
         if request_type == 'fetch':
             # Access protected resource
             response = self.access(
                 credentials, url, params, method, headers, body)
             result = response.content
 
             # Forward status
             adapter.status = str(response.status) + ' ' + str(response.reason)
 
             # Forward headers
             for k, v in response.getheaders():
                 logging.info('    {0}: {1}'.format(k, v))
                 adapter.set_header(k, v)
 
         elif request_type == 'elements':
             # Create request elements
             if json_input:
                 result = self.request_elements(
                     json_input=json_input, return_json=True)
             else:
                 result = self.request_elements(credentials=credentials,
                                                url=url,
                                                method=method,
                                                params=params,
                                                headers=headers,
                                                body=body,
                                                return_json=True)
 
             adapter.set_header('Content-Type', 'application/json')
         else:
             result = '{"error": "Bad Request!"}'
 
         # Add the authomatic header
         adapter.set_header(AUTHOMATIC_HEADER, request_type)
 
         # Write result to response
         adapter.write(result)

rhodecode/lib/diffs.py

@@ -1,1272 +1,1272 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Set of diffing helpers, previously part of vcs
 """
 
 import os
 import re
 import bz2
 import gzip
 import time
 
 import collections
 import difflib
 import logging
-import cPickle as pickle
-from itertools import tee, imap
+import pickle
+from itertools import tee
 
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.nodes import FileNode, SubModuleNode
 from rhodecode.lib.utils2 import safe_unicode, safe_str
 
 log = logging.getLogger(__name__)
 
 # define max context, a file with more than this numbers of lines is unusable
 # in browser anyway
 MAX_CONTEXT = 20 * 1024
 DEFAULT_CONTEXT = 3
 
 
 def get_diff_context(request):
     return MAX_CONTEXT if request.GET.get('fullcontext', '') == '1' else DEFAULT_CONTEXT
 
 
 def get_diff_whitespace_flag(request):
     return request.GET.get('ignorews', '') == '1'
 
 
 class OPS(object):
     ADD = 'A'
     MOD = 'M'
     DEL = 'D'
 
 
 def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True, context=3):
     """
     Returns git style diff between given ``filenode_old`` and ``filenode_new``.
 
     :param ignore_whitespace: ignore whitespaces in diff
     """
     # make sure we pass in default context
     context = context or 3
     # protect against IntOverflow when passing HUGE context
     if context > MAX_CONTEXT:
         context = MAX_CONTEXT
 
     submodules = filter(lambda o: isinstance(o, SubModuleNode),
                         [filenode_new, filenode_old])
     if submodules:
         return ''
 
     for filenode in (filenode_old, filenode_new):
         if not isinstance(filenode, FileNode):
             raise VCSError(
                 "Given object should be FileNode object, not %s"
                 % filenode.__class__)
 
     repo = filenode_new.commit.repository
     old_commit = filenode_old.commit or repo.EMPTY_COMMIT
     new_commit = filenode_new.commit
 
     vcs_gitdiff = repo.get_diff(
         old_commit, new_commit, filenode_new.path,
         ignore_whitespace, context, path1=filenode_old.path)
     return vcs_gitdiff
 
 NEW_FILENODE = 1
 DEL_FILENODE = 2
 MOD_FILENODE = 3
 RENAMED_FILENODE = 4
 COPIED_FILENODE = 5
 CHMOD_FILENODE = 6
 BIN_FILENODE = 7
 
 
 class LimitedDiffContainer(object):
 
     def __init__(self, diff_limit, cur_diff_size, diff):
         self.diff = diff
         self.diff_limit = diff_limit
         self.cur_diff_size = cur_diff_size
 
     def __getitem__(self, key):
         return self.diff.__getitem__(key)
 
     def __iter__(self):
         for l in self.diff:
             yield l
 
 
 class Action(object):
     """
     Contains constants for the action value of the lines in a parsed diff.
     """
 
     ADD = 'add'
     DELETE = 'del'
     UNMODIFIED = 'unmod'
 
     CONTEXT = 'context'
     OLD_NO_NL = 'old-no-nl'
     NEW_NO_NL = 'new-no-nl'
 
 
 class DiffProcessor(object):
     """
     Give it a unified or git diff and it returns a list of the files that were
     mentioned in the diff together with a dict of meta information that
     can be used to render it in a HTML template.
 
     .. note:: Unicode handling
 
        The original diffs are a byte sequence and can contain filenames
        in mixed encodings. This class generally returns `unicode` objects
        since the result is intended for presentation to the user.
 
     """
     _chunk_re = re.compile(r'^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')
     _newline_marker = re.compile(r'^\\ No newline at end of file')
 
     # used for inline highlighter word split
     _token_re = re.compile(r'()(&gt;|&lt;|&amp;|\W+?)')
 
     # collapse ranges of commits over given number
     _collapse_commits_over = 5
 
     def __init__(self, diff, format='gitdiff', diff_limit=None,
                  file_limit=None, show_full_diff=True):
         """
         :param diff: A `Diff` object representing a diff from a vcs backend
         :param format: format of diff passed, `udiff` or `gitdiff`
         :param diff_limit: define the size of diff that is considered "big"
             based on that parameter cut off will be triggered, set to None
             to show full diff
         """
         self._diff = diff
         self._format = format
         self.adds = 0
         self.removes = 0
         # calculate diff size
         self.diff_limit = diff_limit
         self.file_limit = file_limit
         self.show_full_diff = show_full_diff
         self.cur_diff_size = 0
         self.parsed = False
         self.parsed_diff = []
 
         log.debug('Initialized DiffProcessor with %s mode', format)
         if format == 'gitdiff':
             self.differ = self._highlight_line_difflib
             self._parser = self._parse_gitdiff
         else:
             self.differ = self._highlight_line_udiff
             self._parser = self._new_parse_gitdiff
 
     def _copy_iterator(self):
         """
         make a fresh copy of generator, we should not iterate thru
         an original as it's needed for repeating operations on
         this instance of DiffProcessor
         """
         self.__udiff, iterator_copy = tee(self.__udiff)
         return iterator_copy
 
     def _escaper(self, string):
         """
         Escaper for diff escapes special chars and checks the diff limit
 
         :param string:
         """
         self.cur_diff_size += len(string)
 
         if not self.show_full_diff and (self.cur_diff_size > self.diff_limit):
             raise DiffLimitExceeded('Diff Limit Exceeded')
 
         return string \
             .replace('&', '&amp;')\
             .replace('<', '&lt;')\
             .replace('>', '&gt;')
 
     def _line_counter(self, l):
         """
         Checks each line and bumps total adds/removes for this diff
 
         :param l:
         """
         if l.startswith('+') and not l.startswith('+++'):
             self.adds += 1
         elif l.startswith('-') and not l.startswith('---'):
             self.removes += 1
         return safe_unicode(l)
 
     def _highlight_line_difflib(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
 
         if line['action'] == Action.DELETE:
             old, new = line, next_
         else:
             old, new = next_, line
 
         oldwords = self._token_re.split(old['line'])
         newwords = self._token_re.split(new['line'])
         sequence = difflib.SequenceMatcher(None, oldwords, newwords)
 
         oldfragments, newfragments = [], []
         for tag, i1, i2, j1, j2 in sequence.get_opcodes():
             oldfrag = ''.join(oldwords[i1:i2])
             newfrag = ''.join(newwords[j1:j2])
             if tag != 'equal':
                 if oldfrag:
                     oldfrag = '<del>%s</del>' % oldfrag
                 if newfrag:
                     newfrag = '<ins>%s</ins>' % newfrag
             oldfragments.append(oldfrag)
             newfragments.append(newfrag)
 
         old['line'] = "".join(oldfragments)
         new['line'] = "".join(newfragments)
 
     def _highlight_line_udiff(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
         start = 0
         limit = min(len(line['line']), len(next_['line']))
         while start < limit and line['line'][start] == next_['line'][start]:
             start += 1
         end = -1
         limit -= start
         while -end <= limit and line['line'][end] == next_['line'][end]:
             end -= 1
         end += 1
         if start or end:
             def do(l):
                 last = end + len(l['line'])
                 if l['action'] == Action.ADD:
                     tag = 'ins'
                 else:
                     tag = 'del'
                 l['line'] = '%s<%s>%s</%s>%s' % (
                     l['line'][:start],
                     tag,
                     l['line'][start:last],
                     tag,
                     l['line'][last:]
                 )
             do(line)
             do(next_)
 
     def _clean_line(self, line, command):
         if command in ['+', '-', ' ']:
             # only modify the line if it's actually a diff thing
             line = line[1:]
         return line
 
     def _parse_gitdiff(self, inline_diff=True):
         _files = []
         diff_container = lambda arg: arg
 
         for chunk in self._diff.chunks():
             head = chunk.header
 
             diff = imap(self._escaper, self.diff_splitter(chunk.diff))
             raw_diff = chunk.raw
             limited_diff = False
             exceeds_limit = False
 
             op = None
             stats = {
                 'added': 0,
                 'deleted': 0,
                 'binary': False,
                 'ops': {},
             }
 
             if head['deleted_file_mode']:
                 op = OPS.DEL
                 stats['binary'] = True
                 stats['ops'][DEL_FILENODE] = 'deleted file'
 
             elif head['new_file_mode']:
                 op = OPS.ADD
                 stats['binary'] = True
                 stats['ops'][NEW_FILENODE] = 'new file %s' % head['new_file_mode']
             else:  # modify operation, can be copy, rename or chmod
 
                 # CHMOD
                 if head['new_mode'] and head['old_mode']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][CHMOD_FILENODE] = (
                         'modified file chmod %s => %s' % (
                             head['old_mode'], head['new_mode']))
                 # RENAME
                 if head['rename_from'] != head['rename_to']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][RENAMED_FILENODE] = (
                         'file renamed from %s to %s' % (
                             head['rename_from'], head['rename_to']))
                 # COPY
                 if head.get('copy_from') and head.get('copy_to'):
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][COPIED_FILENODE] = (
                         'file copied from %s to %s' % (
                             head['copy_from'], head['copy_to']))
 
                 # If our new parsed headers didn't match anything fallback to
                 # old style detection
                 if op is None:
                     if not head['a_file'] and head['b_file']:
                         op = OPS.ADD
                         stats['binary'] = True
                         stats['ops'][NEW_FILENODE] = 'new file'
 
                     elif head['a_file'] and not head['b_file']:
                         op = OPS.DEL
                         stats['binary'] = True
                         stats['ops'][DEL_FILENODE] = 'deleted file'
 
                 # it's not ADD not DELETE
                 if op is None:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][MOD_FILENODE] = 'modified file'
 
             # a real non-binary diff
             if head['a_file'] or head['b_file']:
                 try:
                     raw_diff, chunks, _stats = self._parse_lines(diff)
                     stats['binary'] = False
                     stats['added'] = _stats[0]
                     stats['deleted'] = _stats[1]
                     # explicit mark that it's a modified file
                     if op == OPS.MOD:
                         stats['ops'][MOD_FILENODE] = 'modified file'
                     exceeds_limit = len(raw_diff) > self.file_limit
 
                     # changed from _escaper function so we validate size of
                     # each file instead of the whole diff
                     # diff will hide big files but still show small ones
                     # from my tests, big files are fairly safe to be parsed
                     # but the browser is the bottleneck
                     if not self.show_full_diff and exceeds_limit:
                         raise DiffLimitExceeded('File Limit Exceeded')
 
                 except DiffLimitExceeded:
                     diff_container = lambda _diff: \
                         LimitedDiffContainer(
                             self.diff_limit, self.cur_diff_size, _diff)
 
                     exceeds_limit = len(raw_diff) > self.file_limit
                     limited_diff = True
                     chunks = []
 
             else:  # GIT format binary patch, or possibly empty diff
                 if head['bin_patch']:
                     # we have operation already extracted, but we mark simply
                     # it's a diff we wont show for binary files
                     stats['ops'][BIN_FILENODE] = 'binary diff hidden'
                 chunks = []
 
             if chunks and not self.show_full_diff and op == OPS.DEL:
                 # if not full diff mode show deleted file contents
                 # TODO: anderson: if the view is not too big, there is no way
                 # to see the content of the file
                 chunks = []
 
             chunks.insert(0, [{
                                   'old_lineno': '',
                                   'new_lineno': '',
                                   'action': Action.CONTEXT,
                                   'line': msg,
                               } for _op, msg in stats['ops'].iteritems()
                               if _op not in [MOD_FILENODE]])
 
             _files.append({
                 'filename': safe_unicode(head['b_path']),
                 'old_revision': head['a_blob_id'],
                 'new_revision': head['b_blob_id'],
                 'chunks': chunks,
                 'raw_diff': safe_unicode(raw_diff),
                 'operation': op,
                 'stats': stats,
                 'exceeds_limit': exceeds_limit,
                 'is_limited_diff': limited_diff,
             })
 
         sorter = lambda info: {OPS.ADD: 0, OPS.MOD: 1,
                                OPS.DEL: 2}.get(info['operation'])
 
         if not inline_diff:
             return diff_container(sorted(_files, key=sorter))
 
         # highlight inline changes
         for diff_data in _files:
             for chunk in diff_data['chunks']:
                 lineiter = iter(chunk)
                 try:
                     while 1:
-                        line = lineiter.next()
+                        line = next(lineiter)
                         if line['action'] not in (
                                 Action.UNMODIFIED, Action.CONTEXT):
-                            nextline = lineiter.next()
+                            nextline = next(lineiter)
                             if nextline['action'] in ['unmod', 'context'] or \
                                nextline['action'] == line['action']:
                                 continue
                             self.differ(line, nextline)
                 except StopIteration:
                     pass
 
         return diff_container(sorted(_files, key=sorter))
 
     def _check_large_diff(self):
         if self.diff_limit:
             log.debug('Checking if diff exceeds current diff_limit of %s', self.diff_limit)
         if not self.show_full_diff and (self.cur_diff_size > self.diff_limit):
             raise DiffLimitExceeded('Diff Limit `%s` Exceeded', self.diff_limit)
 
     # FIXME: NEWDIFFS: dan: this replaces _parse_gitdiff
     def _new_parse_gitdiff(self, inline_diff=True):
         _files = []
 
         # this can be overriden later to a LimitedDiffContainer type
         diff_container = lambda arg: arg
 
         for chunk in self._diff.chunks():
             head = chunk.header
             log.debug('parsing diff %r', head)
 
             raw_diff = chunk.raw
             limited_diff = False
             exceeds_limit = False
 
             op = None
             stats = {
                 'added': 0,
                 'deleted': 0,
                 'binary': False,
                 'old_mode': None,
                 'new_mode': None,
                 'ops': {},
             }
             if head['old_mode']:
                 stats['old_mode'] = head['old_mode']
             if head['new_mode']:
                 stats['new_mode'] = head['new_mode']
             if head['b_mode']:
                 stats['new_mode'] = head['b_mode']
 
             # delete file
             if head['deleted_file_mode']:
                 op = OPS.DEL
                 stats['binary'] = True
                 stats['ops'][DEL_FILENODE] = 'deleted file'
 
             # new file
             elif head['new_file_mode']:
                 op = OPS.ADD
                 stats['binary'] = True
                 stats['old_mode'] = None
                 stats['new_mode'] = head['new_file_mode']
                 stats['ops'][NEW_FILENODE] = 'new file %s' % head['new_file_mode']
 
             # modify operation, can be copy, rename or chmod
             else:
                 # CHMOD
                 if head['new_mode'] and head['old_mode']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][CHMOD_FILENODE] = (
                         'modified file chmod %s => %s' % (
                             head['old_mode'], head['new_mode']))
 
                 # RENAME
                 if head['rename_from'] != head['rename_to']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['renamed'] = (head['rename_from'], head['rename_to'])
                     stats['ops'][RENAMED_FILENODE] = (
                         'file renamed from %s to %s' % (
                             head['rename_from'], head['rename_to']))
                 # COPY
                 if head.get('copy_from') and head.get('copy_to'):
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['copied'] = (head['copy_from'], head['copy_to'])
                     stats['ops'][COPIED_FILENODE] = (
                         'file copied from %s to %s' % (
                             head['copy_from'], head['copy_to']))
 
                 # If our new parsed headers didn't match anything fallback to
                 # old style detection
                 if op is None:
                     if not head['a_file'] and head['b_file']:
                         op = OPS.ADD
                         stats['binary'] = True
                         stats['new_file'] = True
                         stats['ops'][NEW_FILENODE] = 'new file'
 
                     elif head['a_file'] and not head['b_file']:
                         op = OPS.DEL
                         stats['binary'] = True
                         stats['ops'][DEL_FILENODE] = 'deleted file'
 
                 # it's not ADD not DELETE
                 if op is None:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][MOD_FILENODE] = 'modified file'
 
             # a real non-binary diff
             if head['a_file'] or head['b_file']:
                 # simulate splitlines, so we keep the line end part
                 diff = self.diff_splitter(chunk.diff)
 
                 # append each file to the diff size
                 raw_chunk_size = len(raw_diff)
 
                 exceeds_limit = raw_chunk_size > self.file_limit
                 self.cur_diff_size += raw_chunk_size
 
                 try:
                     # Check each file instead of the whole diff.
                     # Diff will hide big files but still show small ones.
                     # From the tests big files are fairly safe to be parsed
                     # but the browser is the bottleneck.
                     if not self.show_full_diff and exceeds_limit:
                         log.debug('File `%s` exceeds current file_limit of %s',
                                   safe_unicode(head['b_path']), self.file_limit)
                         raise DiffLimitExceeded(
                             'File Limit %s Exceeded', self.file_limit)
 
                     self._check_large_diff()
 
                     raw_diff, chunks, _stats = self._new_parse_lines(diff)
                     stats['binary'] = False
                     stats['added'] = _stats[0]
                     stats['deleted'] = _stats[1]
                     # explicit mark that it's a modified file
                     if op == OPS.MOD:
                         stats['ops'][MOD_FILENODE] = 'modified file'
 
                 except DiffLimitExceeded:
                     diff_container = lambda _diff: \
                         LimitedDiffContainer(
                             self.diff_limit, self.cur_diff_size, _diff)
 
                     limited_diff = True
                     chunks = []
 
             else:  # GIT format binary patch, or possibly empty diff
                 if head['bin_patch']:
                     # we have operation already extracted, but we mark simply
                     # it's a diff we wont show for binary files
                     stats['ops'][BIN_FILENODE] = 'binary diff hidden'
                 chunks = []
 
             # Hide content of deleted node by setting empty chunks
             if chunks and not self.show_full_diff and op == OPS.DEL:
                 # if not full diff mode show deleted file contents
                 # TODO: anderson: if the view is not too big, there is no way
                 # to see the content of the file
                 chunks = []
 
             chunks.insert(
                 0, [{'old_lineno': '',
                      'new_lineno': '',
                      'action': Action.CONTEXT,
                      'line': msg,
                      } for _op, msg in stats['ops'].iteritems()
                     if _op not in [MOD_FILENODE]])
 
             original_filename = safe_unicode(head['a_path'])
             _files.append({
                 'original_filename': original_filename,
                 'filename': safe_unicode(head['b_path']),
                 'old_revision': head['a_blob_id'],
                 'new_revision': head['b_blob_id'],
                 'chunks': chunks,
                 'raw_diff': safe_unicode(raw_diff),
                 'operation': op,
                 'stats': stats,
                 'exceeds_limit': exceeds_limit,
                 'is_limited_diff': limited_diff,
             })
 
         sorter = lambda info: {OPS.ADD: 0, OPS.MOD: 1,
                                OPS.DEL: 2}.get(info['operation'])
 
         return diff_container(sorted(_files, key=sorter))
 
     # FIXME: NEWDIFFS: dan: this gets replaced by _new_parse_lines
     def _parse_lines(self, diff_iter):
         """
         Parse the diff an return data for the template.
         """
 
         stats = [0, 0]
         chunks = []
         raw_diff = []
 
         try:
-            line = diff_iter.next()
+            line = next(diff_iter)
 
             while line:
                 raw_diff.append(line)
                 lines = []
                 chunks.append(lines)
 
                 match = self._chunk_re.match(line)
 
                 if not match:
                     break
 
                 gr = match.groups()
                 (old_line, old_end,
                  new_line, new_end) = [int(x or 1) for x in gr[:-1]]
                 old_line -= 1
                 new_line -= 1
 
                 context = len(gr) == 5
                 old_end += old_line
                 new_end += new_line
 
                 if context:
                     # skip context only if it's first line
                     if int(gr[0]) > 1:
                         lines.append({
                             'old_lineno': '...',
                             'new_lineno': '...',
                             'action':     Action.CONTEXT,
                             'line':       line,
                         })
 
-                line = diff_iter.next()
+                line = next(diff_iter)
 
                 while old_line < old_end or new_line < new_end:
                     command = ' '
                     if line:
                         command = line[0]
 
                     affects_old = affects_new = False
 
                     # ignore those if we don't expect them
                     if command in '#@':
                         continue
                     elif command == '+':
                         affects_new = True
                         action = Action.ADD
                         stats[0] += 1
                     elif command == '-':
                         affects_old = True
                         action = Action.DELETE
                         stats[1] += 1
                     else:
                         affects_old = affects_new = True
                         action = Action.UNMODIFIED
 
                     if not self._newline_marker.match(line):
                         old_line += affects_old
                         new_line += affects_new
                         lines.append({
                             'old_lineno':   affects_old and old_line or '',
                             'new_lineno':   affects_new and new_line or '',
                             'action':       action,
                             'line':         self._clean_line(line, command)
                         })
                         raw_diff.append(line)
 
-                    line = diff_iter.next()
+                    line = next(diff_iter)
 
                     if self._newline_marker.match(line):
                         # we need to append to lines, since this is not
                         # counted in the line specs of diff
                         lines.append({
                             'old_lineno':   '...',
                             'new_lineno':   '...',
                             'action':       Action.CONTEXT,
                             'line':         self._clean_line(line, command)
                         })
 
         except StopIteration:
             pass
         return ''.join(raw_diff), chunks, stats
 
     # FIXME: NEWDIFFS: dan: this replaces _parse_lines
     def _new_parse_lines(self, diff_iter):
         """
         Parse the diff an return data for the template.
         """
 
         stats = [0, 0]
         chunks = []
         raw_diff = []
 
         try:
-            line = diff_iter.next()
+            line = next(diff_iter)
 
             while line:
                 raw_diff.append(line)
                 # match header e.g @@ -0,0 +1 @@\n'
                 match = self._chunk_re.match(line)
 
                 if not match:
                     break
 
                 gr = match.groups()
                 (old_line, old_end,
                  new_line, new_end) = [int(x or 1) for x in gr[:-1]]
 
                 lines = []
                 hunk = {
                     'section_header': gr[-1],
                     'source_start': old_line,
                     'source_length': old_end,
                     'target_start': new_line,
                     'target_length': new_end,
                     'lines': lines,
                 }
                 chunks.append(hunk)
 
                 old_line -= 1
                 new_line -= 1
 
                 context = len(gr) == 5
                 old_end += old_line
                 new_end += new_line
 
-                line = diff_iter.next()
+                line = next(diff_iter)
 
                 while old_line < old_end or new_line < new_end:
                     command = ' '
                     if line:
                         command = line[0]
 
                     affects_old = affects_new = False
 
                     # ignore those if we don't expect them
                     if command in '#@':
                         continue
                     elif command == '+':
                         affects_new = True
                         action = Action.ADD
                         stats[0] += 1
                     elif command == '-':
                         affects_old = True
                         action = Action.DELETE
                         stats[1] += 1
                     else:
                         affects_old = affects_new = True
                         action = Action.UNMODIFIED
 
                     if not self._newline_marker.match(line):
                         old_line += affects_old
                         new_line += affects_new
                         lines.append({
                             'old_lineno':   affects_old and old_line or '',
                             'new_lineno':   affects_new and new_line or '',
                             'action':       action,
                             'line':         self._clean_line(line, command)
                         })
                     raw_diff.append(line)
 
-                    line = diff_iter.next()
+                    line = next(diff_iter)
 
                     if self._newline_marker.match(line):
                         # we need to append to lines, since this is not
                         # counted in the line specs of diff
                         if affects_old:
                             action = Action.OLD_NO_NL
                         elif affects_new:
                             action = Action.NEW_NO_NL
                         else:
                             raise Exception('invalid context for no newline')
 
                         lines.append({
                             'old_lineno':   None,
                             'new_lineno':   None,
                             'action':       action,
                             'line':         self._clean_line(line, command)
                         })
 
         except StopIteration:
             pass
 
         return ''.join(raw_diff), chunks, stats
 
     def _safe_id(self, idstring):
         """Make a string safe for including in an id attribute.
 
         The HTML spec says that id attributes 'must begin with
         a letter ([A-Za-z]) and may be followed by any number
         of letters, digits ([0-9]), hyphens ("-"), underscores
         ("_"), colons (":"), and periods (".")'. These regexps
         are slightly over-zealous, in that they remove colons
         and periods unnecessarily.
 
         Whitespace is transformed into underscores, and then
         anything which is not a hyphen or a character that
         matches \w (alphanumerics and underscore) is removed.
 
         """
         # Transform all whitespace to underscore
         idstring = re.sub(r'\s', "_", '%s' % idstring)
         # Remove everything that is not a hyphen or a member of \w
         idstring = re.sub(r'(?!-)\W', "", idstring).lower()
         return idstring
 
     @classmethod
     def diff_splitter(cls, string):
         """
         Diff split that emulates .splitlines() but works only on \n
         """
         if not string:
             return
         elif string == '\n':
             yield u'\n'
         else:
 
             has_newline = string.endswith('\n')
             elements = string.split('\n')
             if has_newline:
                 # skip last element as it's empty string from newlines
                 elements = elements[:-1]
 
             len_elements = len(elements)
 
             for cnt, line in enumerate(elements, start=1):
                 last_line = cnt == len_elements
                 if last_line and not has_newline:
                     yield safe_unicode(line)
                 else:
                     yield safe_unicode(line) + '\n'
 
     def prepare(self, inline_diff=True):
         """
         Prepare the passed udiff for HTML rendering.
 
         :return: A list of dicts with diff information.
         """
         parsed = self._parser(inline_diff=inline_diff)
         self.parsed = True
         self.parsed_diff = parsed
         return parsed
 
     def as_raw(self, diff_lines=None):
         """
         Returns raw diff as a byte string
         """
         return self._diff.raw
 
     def as_html(self, table_class='code-difftable', line_class='line',
                 old_lineno_class='lineno old', new_lineno_class='lineno new',
                 code_class='code', enable_comments=False, parsed_lines=None):
         """
         Return given diff as html table with customized css classes
         """
         # TODO(marcink): not sure how to pass in translator
         # here in an efficient way, leave the _ for proper gettext extraction
         _ = lambda s: s
 
         def _link_to_if(condition, label, url):
             """
             Generates a link if condition is meet or just the label if not.
             """
 
             if condition:
                 return '''<a href="%(url)s" class="tooltip"
                 title="%(title)s">%(label)s</a>''' % {
                     'title': _('Click to select line'),
                     'url': url,
                     'label': label
                 }
             else:
                 return label
         if not self.parsed:
             self.prepare()
 
         diff_lines = self.parsed_diff
         if parsed_lines:
             diff_lines = parsed_lines
 
         _html_empty = True
         _html = []
         _html.append('''<table class="%(table_class)s">\n''' % {
             'table_class': table_class
         })
 
         for diff in diff_lines:
             for line in diff['chunks']:
                 _html_empty = False
                 for change in line:
                     _html.append('''<tr class="%(lc)s %(action)s">\n''' % {
                         'lc': line_class,
                         'action': change['action']
                     })
                     anchor_old_id = ''
                     anchor_new_id = ''
                     anchor_old = "%(filename)s_o%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['old_lineno']
                     }
                     anchor_new = "%(filename)s_n%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['new_lineno']
                     }
                     cond_old = (change['old_lineno'] != '...' and
                                 change['old_lineno'])
                     cond_new = (change['new_lineno'] != '...' and
                                 change['new_lineno'])
                     if cond_old:
                         anchor_old_id = 'id="%s"' % anchor_old
                     if cond_new:
                         anchor_new_id = 'id="%s"' % anchor_new
 
                     if change['action'] != Action.CONTEXT:
                         anchor_link = True
                     else:
                         anchor_link = False
 
                     ###########################################################
                     # COMMENT ICONS
                     ###########################################################
                     _html.append('''\t<td class="add-comment-line"><span class="add-comment-content">''')
 
                     if enable_comments and change['action'] != Action.CONTEXT:
                         _html.append('''<a href="#"><span class="icon-comment-add"></span></a>''')
 
                     _html.append('''</span></td><td class="comment-toggle tooltip" title="Toggle Comment Thread"><i class="icon-comment"></i></td>\n''')
 
                     ###########################################################
                     # OLD LINE NUMBER
                     ###########################################################
                     _html.append('''\t<td %(a_id)s class="%(olc)s">''' % {
                         'a_id': anchor_old_id,
                         'olc': old_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(anchor_link, change['old_lineno'],
                                             '#%s' % anchor_old)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # NEW LINE NUMBER
                     ###########################################################
 
                     _html.append('''\t<td %(a_id)s class="%(nlc)s">''' % {
                         'a_id': anchor_new_id,
                         'nlc': new_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(anchor_link, change['new_lineno'],
                                             '#%s' % anchor_new)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # CODE
                     ###########################################################
                     code_classes = [code_class]
                     if (not enable_comments or
                             change['action'] == Action.CONTEXT):
                         code_classes.append('no-comment')
                     _html.append('\t<td class="%s">' % ' '.join(code_classes))
                     _html.append('''\n\t\t<pre>%(code)s</pre>\n''' % {
                         'code': change['line']
                     })
 
                     _html.append('''\t</td>''')
                     _html.append('''\n</tr>\n''')
         _html.append('''</table>''')
         if _html_empty:
             return None
         return ''.join(_html)
 
     def stat(self):
         """
         Returns tuple of added, and removed lines for this instance
         """
         return self.adds, self.removes
 
     def get_context_of_line(
             self, path, diff_line=None, context_before=3, context_after=3):
         """
         Returns the context lines for the specified diff line.
 
         :type diff_line: :class:`DiffLineNumber`
         """
         assert self.parsed, "DiffProcessor is not initialized."
 
         if None not in diff_line:
             raise ValueError(
                 "Cannot specify both line numbers: {}".format(diff_line))
 
         file_diff = self._get_file_diff(path)
         chunk, idx = self._find_chunk_line_index(file_diff, diff_line)
 
         first_line_to_include = max(idx - context_before, 0)
         first_line_after_context = idx + context_after + 1
         context_lines = chunk[first_line_to_include:first_line_after_context]
 
         line_contents = [
             _context_line(line) for line in context_lines
             if _is_diff_content(line)]
         # TODO: johbo: Interim fixup, the diff chunks drop the final newline.
         # Once they are fixed, we can drop this line here.
         if line_contents:
             line_contents[-1] = (
                 line_contents[-1][0], line_contents[-1][1].rstrip('\n') + '\n')
         return line_contents
 
     def find_context(self, path, context, offset=0):
         """
         Finds the given `context` inside of the diff.
 
         Use the parameter `offset` to specify which offset the target line has
         inside of the given `context`. This way the correct diff line will be
         returned.
 
         :param offset: Shall be used to specify the offset of the main line
             within the given `context`.
         """
         if offset < 0 or offset >= len(context):
             raise ValueError(
                 "Only positive values up to the length of the context "
                 "minus one are allowed.")
 
         matches = []
         file_diff = self._get_file_diff(path)
 
         for chunk in file_diff['chunks']:
             context_iter = iter(context)
             for line_idx, line in enumerate(chunk):
                 try:
-                    if _context_line(line) == context_iter.next():
+                    if _context_line(line) == next(context_iter):
                         continue
                 except StopIteration:
                     matches.append((line_idx, chunk))
                 context_iter = iter(context)
 
         # Increment position and triger StopIteration
         # if we had a match at the end
         line_idx += 1
         try:
-            context_iter.next()
+            next(context_iter)
         except StopIteration:
             matches.append((line_idx, chunk))
 
         effective_offset = len(context) - offset
         found_at_diff_lines = [
             _line_to_diff_line_number(chunk[idx - effective_offset])
             for idx, chunk in matches]
 
         return found_at_diff_lines
 
     def _get_file_diff(self, path):
         for file_diff in self.parsed_diff:
             if file_diff['filename'] == path:
                 break
         else:
             raise FileNotInDiffException("File {} not in diff".format(path))
         return file_diff
 
     def _find_chunk_line_index(self, file_diff, diff_line):
         for chunk in file_diff['chunks']:
             for idx, line in enumerate(chunk):
                 if line['old_lineno'] == diff_line.old:
                     return chunk, idx
                 if line['new_lineno'] == diff_line.new:
                     return chunk, idx
         raise LineNotInDiffException(
             "The line {} is not part of the diff.".format(diff_line))
 
 
 def _is_diff_content(line):
     return line['action'] in (
         Action.UNMODIFIED, Action.ADD, Action.DELETE)
 
 
 def _context_line(line):
     return (line['action'], line['line'])
 
 
 DiffLineNumber = collections.namedtuple('DiffLineNumber', ['old', 'new'])
 
 
 def _line_to_diff_line_number(line):
     new_line_no = line['new_lineno'] or None
     old_line_no = line['old_lineno'] or None
     return DiffLineNumber(old=old_line_no, new=new_line_no)
 
 
 class FileNotInDiffException(Exception):
     """
     Raised when the context for a missing file is requested.
 
     If you request the context for a line in a file which is not part of the
     given diff, then this exception is raised.
     """
 
 
 class LineNotInDiffException(Exception):
     """
     Raised when the context for a missing line is requested.
 
     If you request the context for a line in a file and this line is not
     part of the given diff, then this exception is raised.
     """
 
 
 class DiffLimitExceeded(Exception):
     pass
 
 
 # NOTE(marcink): if diffs.mako change, probably this
 # needs a bump to next version
 CURRENT_DIFF_VERSION = 'v5'
 
 
 def _cleanup_cache_file(cached_diff_file):
     # cleanup file to not store it "damaged"
     try:
         os.remove(cached_diff_file)
     except Exception:
         log.exception('Failed to cleanup path %s', cached_diff_file)
 
 
 def _get_compression_mode(cached_diff_file):
     mode = 'bz2'
     if 'mode:plain' in cached_diff_file:
         mode = 'plain'
     elif 'mode:gzip' in cached_diff_file:
         mode = 'gzip'
     return mode
 
 
 def cache_diff(cached_diff_file, diff, commits):
     compression_mode = _get_compression_mode(cached_diff_file)
 
     struct = {
         'version': CURRENT_DIFF_VERSION,
         'diff': diff,
         'commits': commits
     }
 
     start = time.time()
     try:
         if compression_mode == 'plain':
             with open(cached_diff_file, 'wb') as f:
                 pickle.dump(struct, f)
         elif compression_mode == 'gzip':
             with gzip.GzipFile(cached_diff_file, 'wb') as f:
                 pickle.dump(struct, f)
         else:
             with bz2.BZ2File(cached_diff_file, 'wb') as f:
                 pickle.dump(struct, f)
     except Exception:
         log.warn('Failed to save cache', exc_info=True)
         _cleanup_cache_file(cached_diff_file)
 
     log.debug('Saved diff cache under %s in %.4fs', cached_diff_file, time.time() - start)
 
 
 def load_cached_diff(cached_diff_file):
     compression_mode = _get_compression_mode(cached_diff_file)
 
     default_struct = {
         'version': CURRENT_DIFF_VERSION,
         'diff': None,
         'commits': None
     }
 
     has_cache = os.path.isfile(cached_diff_file)
     if not has_cache:
         log.debug('Reading diff cache file failed %s', cached_diff_file)
         return default_struct
 
     data = None
 
     start = time.time()
     try:
         if compression_mode == 'plain':
             with open(cached_diff_file, 'rb') as f:
                 data = pickle.load(f)
         elif compression_mode == 'gzip':
             with gzip.GzipFile(cached_diff_file, 'rb') as f:
                 data = pickle.load(f)
         else:
             with bz2.BZ2File(cached_diff_file, 'rb') as f:
                 data = pickle.load(f)
     except Exception:
         log.warn('Failed to read diff cache file', exc_info=True)
 
     if not data:
         data = default_struct
 
     if not isinstance(data, dict):
         # old version of data ?
         data = default_struct
 
     # check version
     if data.get('version') != CURRENT_DIFF_VERSION:
         # purge cache
         _cleanup_cache_file(cached_diff_file)
         return default_struct
 
     log.debug('Loaded diff cache from %s in %.4fs', cached_diff_file, time.time() - start)
 
     return data
 
 
 def generate_diff_cache_key(*args):
     """
     Helper to generate a cache key using arguments
     """
     def arg_mapper(input_param):
         input_param = safe_str(input_param)
         # we cannot allow '/' in arguments since it would allow
         # subdirectory usage
         input_param.replace('/', '_')
         return input_param or None  # prevent empty string arguments
 
     return '_'.join([
         '{}' for i in range(len(args))]).format(*map(arg_mapper, args))
 
 
 def diff_cache_exist(cache_storage, *args):
     """
     Based on all generated arguments check and return a cache path
     """
     args = list(args) + ['mode:gzip']
     cache_key = generate_diff_cache_key(*args)
     cache_file_path = os.path.join(cache_storage, cache_key)
     # prevent path traversal attacks using some param that have e.g '../../'
     if not os.path.abspath(cache_file_path).startswith(cache_storage):
         raise ValueError('Final path must be within {}'.format(cache_storage))
 
     return cache_file_path