remote-clone: obfuscate also given query string paramas that RhodeCode uses. Fixes #4668

tests/test_main.py

@@ -1,36 +1,48 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2016 RodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import mock
+import pytest
 
 from vcsserver import main
+from vcsserver.base import obfuscate_qs
 
 
 @mock.patch('vcsserver.main.VcsServerCommand', mock.Mock())
 @mock.patch('vcsserver.hgpatches.patch_largefiles_capabilities')
 def test_applies_largefiles_patch(patch_largefiles_capabilities):
     main.main([])
     patch_largefiles_capabilities.assert_called_once_with()
 
 
 @mock.patch('vcsserver.main.VcsServerCommand', mock.Mock())
 @mock.patch('vcsserver.main.MercurialFactory', None)
 @mock.patch(
     'vcsserver.hgpatches.patch_largefiles_capabilities',
     mock.Mock(side_effect=Exception("Must not be called")))
 def test_applies_largefiles_patch_only_if_mercurial_is_available():
     main.main([])
+
+
[e-mail]('given, expected', [
+    ('foo=bar', 'foo=bar'),
+    ('auth_token=secret', 'auth_token=*****'),
+    ('auth_token=secret&api_key=secret2', 'auth_token=*****&api_key=*****'),
+    ('auth_token=secret&api_key=secret2&param=value', 'auth_token=*****&api_key=*****&param=value'),
+])
+def test_obfuscate_qs(given, expected):
+    assert expected == obfuscate_qs(given)

vcsserver/base.py

@@ -1,71 +1,81 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2016 RodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import logging
-
+import urlparse
 
 log = logging.getLogger(__name__)
 
 
 class RepoFactory(object):
     """
     Utility to create instances of repository
 
     It provides internal caching of the `repo` object based on
     the :term:`call context`.
     """
 
     def __init__(self, repo_cache):
         self._cache = repo_cache
 
     def _create_config(self, path, config):
         config = {}
         return config
 
     def _create_repo(self, wire, create):
         raise NotImplementedError()
 
     def repo(self, wire, create=False):
         """
         Get a repository instance for the given path.
 
         Uses internally the low level beaker API since the decorators introduce
         significant overhead.
         """
         def create_new_repo():
             return self._create_repo(wire, create)
 
         return self._repo(wire, create_new_repo)
 
     def _repo(self, wire, createfunc):
         context = wire.get('context', None)
         cache = wire.get('cache', True)
         log.debug(
             'GET %s@%s with cache:%s. Context: %s',
             self.__class__.__name__, wire['path'], cache, context)
 
         if context and cache:
             cache_key = (context, wire['path'])
             log.debug(
                 'FETCH %s@%s repo object from cache. Context: %s',
                 self.__class__.__name__, wire['path'], context)
             return self._cache.get(key=cache_key, createfunc=createfunc)
         else:
             log.debug(
                 'INIT %s@%s repo object based on wire %s. Context: %s',
                 self.__class__.__name__, wire['path'], wire, context)
             return createfunc()
+
+
+def obfuscate_qs(query_string):
+    parsed = []
+    for k, v in urlparse.parse_qsl(query_string):
+        if k in ['auth_token', 'api_key']:
+            v = "*****"
+        parsed.append((k, v))
+
+    return '&'.join('{}={}'.format(k,v) for k,v in parsed)

vcsserver/git.py

@@ -1,579 +1,580 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2016 RodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import logging
 import os
 import posixpath as vcspath
 import re
 import stat
 import urllib
 import urllib2
 from functools import wraps
 
 from dulwich import index, objects
 from dulwich.client import HttpGitClient, LocalGitClient
 from dulwich.errors import (
     NotGitRepository, ChecksumMismatch, WrongObjectException,
     MissingCommitError, ObjectMissing, HangupException,
     UnexpectedCommandError)
 from dulwich.repo import Repo as DulwichRepo, Tag
 from dulwich.server import update_server_info
 
 from vcsserver import exceptions, settings, subprocessio
 from vcsserver.utils import safe_str
-from vcsserver.base import RepoFactory
+from vcsserver.base import RepoFactory, obfuscate_qs
 from vcsserver.hgcompat import (
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
 
 
 DIR_STAT = stat.S_IFDIR
 FILE_MODE = stat.S_IFMT
 GIT_LINK = objects.S_IFGITLINK
 
 log = logging.getLogger(__name__)
 
 
 def reraise_safe_exceptions(func):
     """Converts Dulwich exceptions to something neutral."""
     @wraps(func)
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except (ChecksumMismatch, WrongObjectException, MissingCommitError,
                 ObjectMissing) as e:
             raise exceptions.LookupException(e.message)
         except (HangupException, UnexpectedCommandError) as e:
             raise exceptions.VcsException(e.message)
     return wrapper
 
 
 class Repo(DulwichRepo):
     """
     A wrapper for dulwich Repo class.
 
     Since dulwich is sometimes keeping .idx file descriptors open, it leads to
     "Too many open files" error. We need to close all opened file descriptors
     once the repo object is destroyed.
 
     TODO: mikhail: please check if we need this wrapper after updating dulwich
     to 0.12.0 +
     """
     def __del__(self):
         if hasattr(self, 'object_store'):
             self.close()
 
 
 class GitFactory(RepoFactory):
 
     def _create_repo(self, wire, create):
         repo_path = str_to_dulwich(wire['path'])
         return Repo(repo_path)
 
 
 class GitRemote(object):
 
     def __init__(self, factory):
         self._factory = factory
 
         self._bulk_methods = {
             "author": self.commit_attribute,
             "date": self.get_object_attrs,
             "message": self.commit_attribute,
             "parents": self.commit_attribute,
             "_commit": self.revision,
         }
 
     def _assign_ref(self, wire, ref, commit_id):
         repo = self._factory.repo(wire)
         repo[ref] = commit_id
 
     @reraise_safe_exceptions
     def add_object(self, wire, content):
         repo = self._factory.repo(wire)
         blob = objects.Blob()
         blob.set_raw_string(content)
         repo.object_store.add_object(blob)
         return blob.id
 
     @reraise_safe_exceptions
     def assert_correct_path(self, wire):
         try:
             self._factory.repo(wire)
         except NotGitRepository as e:
             # Exception can contain unicode which we convert
             raise exceptions.AbortException(repr(e))
 
     @reraise_safe_exceptions
     def bare(self, wire):
         repo = self._factory.repo(wire)
         return repo.bare
 
     @reraise_safe_exceptions
     def blob_as_pretty_string(self, wire, sha):
         repo = self._factory.repo(wire)
         return repo[sha].as_pretty_string()
 
     @reraise_safe_exceptions
     def blob_raw_length(self, wire, sha):
         repo = self._factory.repo(wire)
         blob = repo[sha]
         return blob.raw_length()
 
     @reraise_safe_exceptions
     def bulk_request(self, wire, rev, pre_load):
         result = {}
         for attr in pre_load:
             try:
                 method = self._bulk_methods[attr]
                 args = [wire, rev]
                 if attr == "date":
                     args.extend(["commit_time", "commit_timezone"])
                 elif attr in ["author", "message", "parents"]:
                     args.append(attr)
                 result[attr] = method(*args)
             except KeyError:
                 raise exceptions.VcsException(
                     "Unknown bulk attribute: %s" % attr)
         return result
 
     def _build_opener(self, url):
         handlers = []
         url_obj = url_parser(url)
         _, authinfo = url_obj.authinfo()
 
         if authinfo:
             # create a password manager
             passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((httpbasicauthhandler(passmgr),
                              httpdigestauthhandler(passmgr)))
 
         return urllib2.build_opener(*handlers)
 
     @reraise_safe_exceptions
     def check_url(self, url, config):
         url_obj = url_parser(url)
         test_uri, _ = url_obj.authinfo()
         url_obj.passwd = '*****'
+        url_obj.query = obfuscate_qs(url_obj.query)
         cleaned_uri = str(url_obj)
         log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
 
         if not test_uri.endswith('info/refs'):
             test_uri = test_uri.rstrip('/') + '/info/refs'
 
         o = self._build_opener(url)
         o.addheaders = [('User-Agent', 'git/1.7.8.0')]  # fake some git
 
         q = {"service": 'git-upload-pack'}
         qs = '?%s' % urllib.urlencode(q)
         cu = "%s%s" % (test_uri, qs)
         req = urllib2.Request(cu, None, {})
 
         try:
             log.debug("Trying to open URL %s", cleaned_uri)
             resp = o.open(req)
             if resp.code != 200:
                 raise exceptions.URLError('Return Code is not 200')
         except Exception as e:
             log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
             # means it cannot be cloned
             raise exceptions.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
 
         # now detect if it's proper git repo
         gitdata = resp.read()
         if 'service=git-upload-pack' in gitdata:
             pass
         elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
             # old style git can return some other format !
             pass
         else:
             raise exceptions.URLError(
                 "url [%s] does not look like an git" % (cleaned_uri,))
 
         return True
 
     @reraise_safe_exceptions
     def clone(self, wire, url, deferred, valid_refs, update_after_clone):
         remote_refs = self.fetch(wire, url, apply_refs=False)
         repo = self._factory.repo(wire)
         if isinstance(valid_refs, list):
             valid_refs = tuple(valid_refs)
 
         for k in remote_refs:
             # only parse heads/tags and skip so called deferred tags
             if k.startswith(valid_refs) and not k.endswith(deferred):
                 repo[k] = remote_refs[k]
 
         if update_after_clone:
             # we want to checkout HEAD
             repo["HEAD"] = remote_refs["HEAD"]
             index.build_index_from_tree(repo.path, repo.index_path(),
                                         repo.object_store, repo["HEAD"].tree)
 
     # TODO: this is quite complex, check if that can be simplified
     @reraise_safe_exceptions
     def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
         repo = self._factory.repo(wire)
         object_store = repo.object_store
 
         # Create tree and populates it with blobs
         commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
 
         for node in updated:
             # Compute subdirs if needed
             dirpath, nodename = vcspath.split(node['path'])
             dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
             parent = commit_tree
             ancestors = [('', parent)]
 
             # Tries to dig for the deepest existing tree
             while dirnames:
                 curdir = dirnames.pop(0)
                 try:
                     dir_id = parent[curdir][1]
                 except KeyError:
                     # put curdir back into dirnames and stops
                     dirnames.insert(0, curdir)
                     break
                 else:
                     # If found, updates parent
                     parent = repo[dir_id]
                     ancestors.append((curdir, parent))
             # Now parent is deepest existing tree and we need to create
             # subtrees for dirnames (in reverse order)
             # [this only applies for nodes from added]
             new_trees = []
 
             blob = objects.Blob.from_string(node['content'])
 
             if dirnames:
                 # If there are trees which should be created we need to build
                 # them now (in reverse order)
                 reversed_dirnames = list(reversed(dirnames))
                 curtree = objects.Tree()
                 curtree[node['node_path']] = node['mode'], blob.id
                 new_trees.append(curtree)
                 for dirname in reversed_dirnames[:-1]:
                     newtree = objects.Tree()
                     newtree[dirname] = (DIR_STAT, curtree.id)
                     new_trees.append(newtree)
                     curtree = newtree
                 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
             else:
                 parent.add(
                     name=node['node_path'], mode=node['mode'], hexsha=blob.id)
 
             new_trees.append(parent)
             # Update ancestors
             reversed_ancestors = reversed(
                 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
             for parent, tree, path in reversed_ancestors:
                 parent[path] = (DIR_STAT, tree.id)
                 object_store.add_object(tree)
 
             object_store.add_object(blob)
             for tree in new_trees:
                 object_store.add_object(tree)
 
         for node_path in removed:
             paths = node_path.split('/')
             tree = commit_tree
             trees = [tree]
             # Traverse deep into the forest...
             for path in paths:
                 try:
                     obj = repo[tree[path][1]]
                     if isinstance(obj, objects.Tree):
                         trees.append(obj)
                         tree = obj
                 except KeyError:
                     break
             # Cut down the blob and all rotten trees on the way back...
             for path, tree in reversed(zip(paths, trees)):
                 del tree[path]
                 if tree:
                     # This tree still has elements - don't remove it or any
                     # of it's parents
                     break
 
         object_store.add_object(commit_tree)
 
         # Create commit
         commit = objects.Commit()
         commit.tree = commit_tree.id
         for k, v in commit_data.iteritems():
             setattr(commit, k, v)
         object_store.add_object(commit)
 
         ref = 'refs/heads/%s' % branch
         repo.refs[ref] = commit.id
 
         return commit.id
 
     @reraise_safe_exceptions
     def fetch(self, wire, url, apply_refs=True, refs=None):
         if url != 'default' and '://' not in url:
             client = LocalGitClient(url)
         else:
             url_obj = url_parser(url)
             o = self._build_opener(url)
             url, _ = url_obj.authinfo()
             client = HttpGitClient(base_url=url, opener=o)
         repo = self._factory.repo(wire)
 
         determine_wants = repo.object_store.determine_wants_all
         if refs:
             def determine_wants_requested(references):
                 return [references[r] for r in references if r in refs]
             determine_wants = determine_wants_requested
 
         try:
             remote_refs = client.fetch(
                 path=url, target=repo, determine_wants=determine_wants)
         except NotGitRepository:
             log.warning(
                 'Trying to fetch from "%s" failed, not a Git repository.', url)
             raise exceptions.AbortException()
 
         # mikhail: client.fetch() returns all the remote refs, but fetches only
         # refs filtered by `determine_wants` function. We need to filter result
         # as well
         if refs:
             remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
 
         if apply_refs:
             # TODO: johbo: Needs proper test coverage with a git repository
             # that contains a tag object, so that we would end up with
             # a peeled ref at this point.
             PEELED_REF_MARKER = '^{}'
             for k in remote_refs:
                 if k.endswith(PEELED_REF_MARKER):
                     log.info("Skipping peeled reference %s", k)
                     continue
                 repo[k] = remote_refs[k]
 
             if refs:
                 # mikhail: explicitly set the head to the last ref.
                 repo['HEAD'] = remote_refs[refs[-1]]
 
             # TODO: mikhail: should we return remote_refs here to be
             # consistent?
         else:
             return remote_refs
 
     @reraise_safe_exceptions
     def get_remote_refs(self, wire, url):
         repo = Repo(url)
         return repo.get_refs()
 
     @reraise_safe_exceptions
     def get_description(self, wire):
         repo = self._factory.repo(wire)
         return repo.get_description()
 
     @reraise_safe_exceptions
     def get_file_history(self, wire, file_path, commit_id, limit):
         repo = self._factory.repo(wire)
         include = [commit_id]
         paths = [file_path]
 
         walker = repo.get_walker(include, paths=paths, max_entries=limit)
         return [x.commit.id for x in walker]
 
     @reraise_safe_exceptions
     def get_missing_revs(self, wire, rev1, rev2, path2):
         repo = self._factory.repo(wire)
         LocalGitClient(thin_packs=False).fetch(path2, repo)
 
         wire_remote = wire.copy()
         wire_remote['path'] = path2
         repo_remote = self._factory.repo(wire_remote)
         LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
 
         revs = [
             x.commit.id
             for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
         return revs
 
     @reraise_safe_exceptions
     def get_object(self, wire, sha):
         repo = self._factory.repo(wire)
         obj = repo.get_object(sha)
         commit_id = obj.id
 
         if isinstance(obj, Tag):
             commit_id = obj.object[1]
 
         return {
             'id': obj.id,
             'type': obj.type_name,
             'commit_id': commit_id
         }
 
     @reraise_safe_exceptions
     def get_object_attrs(self, wire, sha, *attrs):
         repo = self._factory.repo(wire)
         obj = repo.get_object(sha)
         return list(getattr(obj, a) for a in attrs)
 
     @reraise_safe_exceptions
     def get_refs(self, wire):
         repo = self._factory.repo(wire)
         result = {}
         for ref, sha in repo.refs.as_dict().items():
             peeled_sha = repo.get_peeled(ref)
             result[ref] = peeled_sha
         return result
 
     @reraise_safe_exceptions
     def get_refs_path(self, wire):
         repo = self._factory.repo(wire)
         return repo.refs.path
 
     @reraise_safe_exceptions
     def head(self, wire):
         repo = self._factory.repo(wire)
         return repo.head()
 
     @reraise_safe_exceptions
     def init(self, wire):
         repo_path = str_to_dulwich(wire['path'])
         self.repo = Repo.init(repo_path)
 
     @reraise_safe_exceptions
     def init_bare(self, wire):
         repo_path = str_to_dulwich(wire['path'])
         self.repo = Repo.init_bare(repo_path)
 
     @reraise_safe_exceptions
     def revision(self, wire, rev):
         repo = self._factory.repo(wire)
         obj = repo[rev]
         obj_data = {
             'id': obj.id,
         }
         try:
             obj_data['tree'] = obj.tree
         except AttributeError:
             pass
         return obj_data
 
     @reraise_safe_exceptions
     def commit_attribute(self, wire, rev, attr):
         repo = self._factory.repo(wire)
         obj = repo[rev]
         return getattr(obj, attr)
 
     @reraise_safe_exceptions
     def set_refs(self, wire, key, value):
         repo = self._factory.repo(wire)
         repo.refs[key] = value
 
     @reraise_safe_exceptions
     def remove_ref(self, wire, key):
         repo = self._factory.repo(wire)
         del repo.refs[key]
 
     @reraise_safe_exceptions
     def tree_changes(self, wire, source_id, target_id):
         repo = self._factory.repo(wire)
         source = repo[source_id].tree if source_id else None
         target = repo[target_id].tree
         result = repo.object_store.tree_changes(source, target)
         return list(result)
 
     @reraise_safe_exceptions
     def tree_items(self, wire, tree_id):
         repo = self._factory.repo(wire)
         tree = repo[tree_id]
 
         result = []
         for item in tree.iteritems():
             item_sha = item.sha
             item_mode = item.mode
 
             if FILE_MODE(item_mode) == GIT_LINK:
                 item_type = "link"
             else:
                 item_type = repo[item_sha].type_name
 
             result.append((item.path, item_mode, item_sha, item_type))
         return result
 
     @reraise_safe_exceptions
     def update_server_info(self, wire):
         repo = self._factory.repo(wire)
         update_server_info(repo)
 
     @reraise_safe_exceptions
     def discover_git_version(self):
         stdout, _ = self.run_git_command(
             {}, ['--version'], _bare=True, _safe=True)
         prefix = 'git version'
         if stdout.startswith(prefix):
             stdout = stdout[len(prefix):]
         return stdout.strip()
 
     @reraise_safe_exceptions
     def run_git_command(self, wire, cmd, **opts):
         path = wire.get('path', None)
 
         if path and os.path.isdir(path):
             opts['cwd'] = path
 
         if '_bare' in opts:
             _copts = []
             del opts['_bare']
         else:
             _copts = ['-c', 'core.quotepath=false', ]
         safe_call = False
         if '_safe' in opts:
             # no exc on failure
             del opts['_safe']
             safe_call = True
 
         gitenv = os.environ.copy()
         gitenv.update(opts.pop('extra_env', {}))
         # need to clean fix GIT_DIR !
         if 'GIT_DIR' in gitenv:
             del gitenv['GIT_DIR']
         gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
 
         cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
 
         try:
             _opts = {'env': gitenv, 'shell': False}
             _opts.update(opts)
             p = subprocessio.SubprocessIOChunker(cmd, **_opts)
 
             return ''.join(p), ''.join(p.error)
         except (EnvironmentError, OSError) as err:
             tb_err = ("Couldn't run git command (%s).\n"
                       "Original error was:%s\n" % (cmd, err))
             log.exception(tb_err)
             if safe_call:
                 return '', err
             else:
                 raise exceptions.VcsException(tb_err)
 
 
 def str_to_dulwich(value):
     """
     Dulwich 0.10.1a requires `unicode` objects to be passed in.
     """
     return value.decode(settings.WIRE_ENCODING)

vcsserver/hg.py

@@ -1,721 +1,723 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2016 RodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 import logging
 import stat
 import sys
 import urllib
 import urllib2
 
 from hgext import largefiles, rebase
 from hgext.strip import strip as hgext_strip
 from mercurial import commands
 from mercurial import unionrepo
 
 from vcsserver import exceptions
-from vcsserver.base import RepoFactory
+from vcsserver.base import RepoFactory, obfuscate_qs
 from vcsserver.hgcompat import (
     archival, bin, clone, config as hgconfig, diffopts, hex,
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
     httppeer, localrepository, match, memctx, exchange, memfilectx, nullrev,
     patch, peer, revrange, ui, Abort, LookupError, RepoError, RepoLookupError,
     InterventionRequired, RequirementError)
 
 log = logging.getLogger(__name__)
 
 
 def make_ui_from_config(repo_config):
     baseui = ui.ui()
 
     # clean the baseui object
     baseui._ocfg = hgconfig.config()
     baseui._ucfg = hgconfig.config()
     baseui._tcfg = hgconfig.config()
 
     for section, option, value in repo_config:
         baseui.setconfig(section, option, value)
 
     # make our hgweb quiet so it doesn't print output
     baseui.setconfig('ui', 'quiet', 'true')
 
     # force mercurial to only use 1 thread, otherwise it may try to set a
     # signal in a non-main thread, thus generating a ValueError.
     baseui.setconfig('worker', 'numcpus', 1)
 
     # If there is no config for the largefiles extension, we explicitly disable
     # it here. This overrides settings from repositories hgrc file. Recent
     # mercurial versions enable largefiles in hgrc on clone from largefile
     # repo.
     if not baseui.hasconfig('extensions', 'largefiles'):
         log.debug('Explicitly disable largefiles extension for repo.')
         baseui.setconfig('extensions', 'largefiles', '!')
 
     return baseui
 
 
 def reraise_safe_exceptions(func):
     """Decorator for converting mercurial exceptions to something neutral."""
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except (Abort, InterventionRequired):
             raise_from_original(exceptions.AbortException)
         except RepoLookupError:
             raise_from_original(exceptions.LookupException)
         except RequirementError:
             raise_from_original(exceptions.RequirementException)
         except RepoError:
             raise_from_original(exceptions.VcsException)
         except LookupError:
             raise_from_original(exceptions.LookupException)
         except Exception as e:
             if not hasattr(e, '_vcs_kind'):
                 log.exception("Unhandled exception in hg remote call")
                 raise_from_original(exceptions.UnhandledException)
             raise
     return wrapper
 
 
 def raise_from_original(new_type):
     """
     Raise a new exception type with original args and traceback.
     """
     _, original, traceback = sys.exc_info()
     try:
         raise new_type(*original.args), None, traceback
     finally:
         del traceback
 
 
 class MercurialFactory(RepoFactory):
 
     def _create_config(self, config, hooks=True):
         if not hooks:
             hooks_to_clean = frozenset((
                 'changegroup.repo_size', 'preoutgoing.pre_pull',
                 'outgoing.pull_logger', 'prechangegroup.pre_push'))
             new_config = []
             for section, option, value in config:
                 if section == 'hooks' and option in hooks_to_clean:
                     continue
                 new_config.append((section, option, value))
             config = new_config
 
         baseui = make_ui_from_config(config)
         return baseui
 
     def _create_repo(self, wire, create):
         baseui = self._create_config(wire["config"])
         return localrepository(baseui, wire["path"], create)
 
 
 class HgRemote(object):
 
     def __init__(self, factory):
         self._factory = factory
 
         self._bulk_methods = {
             "affected_files": self.ctx_files,
             "author": self.ctx_user,
             "branch": self.ctx_branch,
             "children": self.ctx_children,
             "date": self.ctx_date,
             "message": self.ctx_description,
             "parents": self.ctx_parents,
             "status": self.ctx_status,
             "_file_paths": self.ctx_list,
         }
 
     @reraise_safe_exceptions
     def discover_hg_version(self):
         from mercurial import util
         return util.version()
 
     @reraise_safe_exceptions
     def archive_repo(self, archive_path, mtime, file_info, kind):
         if kind == "tgz":
             archiver = archival.tarit(archive_path, mtime, "gz")
         elif kind == "tbz2":
             archiver = archival.tarit(archive_path, mtime, "bz2")
         elif kind == 'zip':
             archiver = archival.zipit(archive_path, mtime)
         else:
             raise exceptions.ArchiveException(
                 'Remote does not support: "%s".' % kind)
 
         for f_path, f_mode, f_is_link, f_content in file_info:
             archiver.addfile(f_path, f_mode, f_is_link, f_content)
         archiver.done()
 
     @reraise_safe_exceptions
     def bookmarks(self, wire):
         repo = self._factory.repo(wire)
         return dict(repo._bookmarks)
 
     @reraise_safe_exceptions
     def branches(self, wire, normal, closed):
         repo = self._factory.repo(wire)
         iter_branches = repo.branchmap().iterbranches()
         bt = {}
         for branch_name, _heads, tip, is_closed in iter_branches:
             if normal and not is_closed:
                 bt[branch_name] = tip
             if closed and is_closed:
                 bt[branch_name] = tip
 
         return bt
 
     @reraise_safe_exceptions
     def bulk_request(self, wire, rev, pre_load):
         result = {}
         for attr in pre_load:
             try:
                 method = self._bulk_methods[attr]
                 result[attr] = method(wire, rev)
             except KeyError:
                 raise exceptions.VcsException(
                     'Unknown bulk attribute: "%s"' % attr)
         return result
 
     @reraise_safe_exceptions
     def clone(self, wire, source, dest, update_after_clone=False, hooks=True):
         baseui = self._factory._create_config(wire["config"], hooks=hooks)
         clone(baseui, source, dest, noupdate=not update_after_clone)
 
     @reraise_safe_exceptions
     def commitctx(
             self, wire, message, parents, commit_time, commit_timezone,
             user, files, extra, removed, updated):
 
         def _filectxfn(_repo, memctx, path):
             """
             Marks given path as added/changed/removed in a given _repo. This is
             for internal mercurial commit function.
             """
 
             # check if this path is removed
             if path in removed:
                 # returning None is a way to mark node for removal
                 return None
 
             # check if this path is added
             for node in updated:
                 if node['path'] == path:
                     return memfilectx(
                         _repo,
                         path=node['path'],
                         data=node['content'],
                         islink=False,
                         isexec=bool(node['mode'] & stat.S_IXUSR),
                         copied=False,
                         memctx=memctx)
 
             raise exceptions.AbortException(
                 "Given path haven't been marked as added, "
                 "changed or removed (%s)" % path)
 
         repo = self._factory.repo(wire)
 
         commit_ctx = memctx(
             repo=repo,
             parents=parents,
             text=message,
             files=files,
             filectxfn=_filectxfn,
             user=user,
             date=(commit_time, commit_timezone),
             extra=extra)
 
         n = repo.commitctx(commit_ctx)
         new_id = hex(n)
 
         return new_id
 
     @reraise_safe_exceptions
     def ctx_branch(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.branch()
 
     @reraise_safe_exceptions
     def ctx_children(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return [child.rev() for child in ctx.children()]
 
     @reraise_safe_exceptions
     def ctx_date(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.date()
 
     @reraise_safe_exceptions
     def ctx_description(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.description()
 
     @reraise_safe_exceptions
     def ctx_diff(
             self, wire, revision, git=True, ignore_whitespace=True, context=3):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         result = ctx.diff(
             git=git, ignore_whitespace=ignore_whitespace, context=context)
         return list(result)
 
     @reraise_safe_exceptions
     def ctx_files(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.files()
 
     @reraise_safe_exceptions
     def ctx_list(self, path, revision):
         repo = self._factory.repo(path)
         ctx = repo[revision]
         return list(ctx)
 
     @reraise_safe_exceptions
     def ctx_parents(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return [parent.rev() for parent in ctx.parents()]
 
     @reraise_safe_exceptions
     def ctx_substate(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.substate
 
     @reraise_safe_exceptions
     def ctx_status(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         status = repo[ctx.p1().node()].status(other=ctx.node())
         # object of status (odd, custom named tuple in mercurial) is not
         # correctly serializable via Pyro, we make it a list, as the underling
         # API expects this to be a list
         return list(status)
 
     @reraise_safe_exceptions
     def ctx_user(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.user()
 
     @reraise_safe_exceptions
     def check_url(self, url, config):
         _proto = None
         if '+' in url[:url.find('://')]:
             _proto = url[0:url.find('+')]
             url = url[url.find('+') + 1:]
         handlers = []
         url_obj = url_parser(url)
         test_uri, authinfo = url_obj.authinfo()
         url_obj.passwd = '*****'
+        url_obj.query = obfuscate_qs(url_obj.query)
+
         cleaned_uri = str(url_obj)
         log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
 
         if authinfo:
             # create a password manager
             passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((httpbasicauthhandler(passmgr),
                              httpdigestauthhandler(passmgr)))
 
         o = urllib2.build_opener(*handlers)
         o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
                         ('Accept', 'application/mercurial-0.1')]
 
         q = {"cmd": 'between'}
         q.update({'pairs': "%s-%s" % ('0' * 40, '0' * 40)})
         qs = '?%s' % urllib.urlencode(q)
         cu = "%s%s" % (test_uri, qs)
         req = urllib2.Request(cu, None, {})
 
         try:
             log.debug("Trying to open URL %s", cleaned_uri)
             resp = o.open(req)
             if resp.code != 200:
                 raise exceptions.URLError('Return Code is not 200')
         except Exception as e:
             log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
             # means it cannot be cloned
             raise exceptions.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
 
         # now check if it's a proper hg repo, but don't do it for svn
         try:
             if _proto == 'svn':
                 pass
             else:
                 # check for pure hg repos
                 log.debug(
                     "Verifying if URL is a Mercurial repository: %s",
                     cleaned_uri)
                 httppeer(make_ui_from_config(config), url).lookup('tip')
         except Exception as e:
             log.warning("URL is not a valid Mercurial repository: %s",
                         cleaned_uri)
             raise exceptions.URLError(
                 "url [%s] does not look like an hg repo org_exc: %s"
                 % (cleaned_uri, e))
 
         log.info("URL is a valid Mercurial repository: %s", cleaned_uri)
         return True
 
     @reraise_safe_exceptions
     def diff(
             self, wire, rev1, rev2, file_filter, opt_git, opt_ignorews,
             context):
         repo = self._factory.repo(wire)
 
         if file_filter:
             filter = match(file_filter[0], '', [file_filter[1]])
         else:
             filter = file_filter
         opts = diffopts(git=opt_git, ignorews=opt_ignorews, context=context)
 
         try:
             return "".join(patch.diff(
                 repo, node1=rev1, node2=rev2, match=filter, opts=opts))
         except RepoLookupError:
             raise exceptions.LookupException()
 
     @reraise_safe_exceptions
     def file_history(self, wire, revision, path, limit):
         repo = self._factory.repo(wire)
 
         ctx = repo[revision]
         fctx = ctx.filectx(path)
 
         def history_iter():
             limit_rev = fctx.rev()
             for obj in reversed(list(fctx.filelog())):
                 obj = fctx.filectx(obj)
                 if limit_rev >= obj.rev():
                     yield obj
 
         history = []
         for cnt, obj in enumerate(history_iter()):
             if limit and cnt >= limit:
                 break
             history.append(hex(obj.node()))
 
         return [x for x in history]
 
     @reraise_safe_exceptions
     def file_history_untill(self, wire, revision, path, limit):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         fctx = ctx.filectx(path)
 
         file_log = list(fctx.filelog())
         if limit:
             # Limit to the last n items
             file_log = file_log[-limit:]
 
         return [hex(fctx.filectx(cs).node()) for cs in reversed(file_log)]
 
     @reraise_safe_exceptions
     def fctx_annotate(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         fctx = ctx.filectx(path)
 
         result = []
         for i, annotate_data in enumerate(fctx.annotate()):
             ln_no = i + 1
             sha = hex(annotate_data[0].node())
             result.append((ln_no, sha, annotate_data[1]))
         return result
 
     @reraise_safe_exceptions
     def fctx_data(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         fctx = ctx.filectx(path)
         return fctx.data()
 
     @reraise_safe_exceptions
     def fctx_flags(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         fctx = ctx.filectx(path)
         return fctx.flags()
 
     @reraise_safe_exceptions
     def fctx_size(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         fctx = ctx.filectx(path)
         return fctx.size()
 
     @reraise_safe_exceptions
     def get_all_commit_ids(self, wire, name):
         repo = self._factory.repo(wire)
         revs = repo.filtered(name).changelog.index
         return map(lambda x: hex(x[7]), revs)[:-1]
 
     @reraise_safe_exceptions
     def get_config_value(self, wire, section, name, untrusted=False):
         repo = self._factory.repo(wire)
         return repo.ui.config(section, name, untrusted=untrusted)
 
     @reraise_safe_exceptions
     def get_config_bool(self, wire, section, name, untrusted=False):
         repo = self._factory.repo(wire)
         return repo.ui.configbool(section, name, untrusted=untrusted)
 
     @reraise_safe_exceptions
     def get_config_list(self, wire, section, name, untrusted=False):
         repo = self._factory.repo(wire)
         return repo.ui.configlist(section, name, untrusted=untrusted)
 
     @reraise_safe_exceptions
     def is_large_file(self, wire, path):
         return largefiles.lfutil.isstandin(path)
 
     @reraise_safe_exceptions
     def in_store(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.instore(repo, sha)
 
     @reraise_safe_exceptions
     def in_user_cache(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.inusercache(repo.ui, sha)
 
     @reraise_safe_exceptions
     def store_path(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.storepath(repo, sha)
 
     @reraise_safe_exceptions
     def link(self, wire, sha, path):
         repo = self._factory.repo(wire)
         largefiles.lfutil.link(
             largefiles.lfutil.usercachepath(repo.ui, sha), path)
 
     @reraise_safe_exceptions
     def localrepository(self, wire, create=False):
         self._factory.repo(wire, create=create)
 
     @reraise_safe_exceptions
     def lookup(self, wire, revision, both):
         # TODO Paris: Ugly hack to "deserialize" long for msgpack
         if isinstance(revision, float):
             revision = long(revision)
         repo = self._factory.repo(wire)
         try:
             ctx = repo[revision]
         except RepoLookupError:
             raise exceptions.LookupException(revision)
         except LookupError as e:
             raise exceptions.LookupException(e.name)
 
         if not both:
             return ctx.hex()
 
         ctx = repo[ctx.hex()]
         return ctx.hex(), ctx.rev()
 
     @reraise_safe_exceptions
     def pull(self, wire, url, commit_ids=None):
         repo = self._factory.repo(wire)
         remote = peer(repo, {}, url)
         if commit_ids:
             commit_ids = [bin(commit_id) for commit_id in commit_ids]
 
         return exchange.pull(
             repo, remote, heads=commit_ids, force=None).cgresult
 
     @reraise_safe_exceptions
     def revision(self, wire, rev):
         repo = self._factory.repo(wire)
         ctx = repo[rev]
         return ctx.rev()
 
     @reraise_safe_exceptions
     def rev_range(self, wire, filter):
         repo = self._factory.repo(wire)
         revisions = [rev for rev in revrange(repo, filter)]
         return revisions
 
     @reraise_safe_exceptions
     def rev_range_hash(self, wire, node):
         repo = self._factory.repo(wire)
 
         def get_revs(repo, rev_opt):
             if rev_opt:
                 revs = revrange(repo, rev_opt)
                 if len(revs) == 0:
                     return (nullrev, nullrev)
                 return max(revs), min(revs)
             else:
                 return len(repo) - 1, 0
 
         stop, start = get_revs(repo, [node + ':'])
         revs = [hex(repo[r].node()) for r in xrange(start, stop + 1)]
         return revs
 
     @reraise_safe_exceptions
     def revs_from_revspec(self, wire, rev_spec, *args, **kwargs):
         other_path = kwargs.pop('other_path', None)
 
         # case when we want to compare two independent repositories
         if other_path and other_path != wire["path"]:
             baseui = self._factory._create_config(wire["config"])
             repo = unionrepo.unionrepository(baseui, other_path, wire["path"])
         else:
             repo = self._factory.repo(wire)
         return list(repo.revs(rev_spec, *args))
 
     @reraise_safe_exceptions
     def strip(self, wire, revision, update, backup):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         hgext_strip(
             repo.baseui, repo, ctx.node(), update=update, backup=backup)
 
     @reraise_safe_exceptions
     def tag(self, wire, name, revision, message, local, user,
             tag_time, tag_timezone):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         node = ctx.node()
 
         date = (tag_time, tag_timezone)
         try:
             repo.tag(name, node, message, local, user, date)
         except Abort:
             log.exception("Tag operation aborted")
             raise exceptions.AbortException()
 
     @reraise_safe_exceptions
     def tags(self, wire):
         repo = self._factory.repo(wire)
         return repo.tags()
 
     @reraise_safe_exceptions
     def update(self, wire, node=None, clean=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         commands.update(baseui, repo, node=node, clean=clean)
 
     @reraise_safe_exceptions
     def identify(self, wire):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         output = io.BytesIO()
         baseui.write = output.write
         # This is required to get a full node id
         baseui.debugflag = True
         commands.identify(baseui, repo, id=True)
 
         return output.getvalue()
 
     @reraise_safe_exceptions
     def pull_cmd(self, wire, source, bookmark=None, branch=None, revision=None,
                  hooks=True):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'], hooks=hooks)
 
         # Mercurial internally has a lot of logic that checks ONLY if
         # option is defined, we just pass those if they are defined then
         opts = {}
         if bookmark:
             opts['bookmark'] = bookmark
         if branch:
             opts['branch'] = branch
         if revision:
             opts['rev'] = revision
 
         commands.pull(baseui, repo, source, **opts)
 
     @reraise_safe_exceptions
     def heads(self, wire, branch=None):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         output = io.BytesIO()
 
         def write(data, **unused_kwargs):
             output.write(data)
 
         baseui.write = write
         if branch:
             args = [branch]
         else:
             args = []
         commands.heads(baseui, repo, template='{node} ', *args)
 
         return output.getvalue()
 
     @reraise_safe_exceptions
     def ancestor(self, wire, revision1, revision2):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         output = io.BytesIO()
         baseui.write = output.write
         commands.debugancestor(baseui, repo, revision1, revision2)
 
         return output.getvalue()
 
     @reraise_safe_exceptions
     def push(self, wire, revisions, dest_path, hooks=True,
              push_branches=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'], hooks=hooks)
         commands.push(baseui, repo, dest=dest_path, rev=revisions,
                       new_branch=push_branches)
 
     @reraise_safe_exceptions
     def merge(self, wire, revision):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig('ui', 'merge', 'internal:dump')
 
         # In case of sub repositories are used mercurial prompts the user in
         # case of merge conflicts or different sub repository sources. By
         # setting the interactive flag to `False` mercurial doesn't prompt the
         # used but instead uses a default value.
         repo.ui.setconfig('ui', 'interactive', False)
 
         commands.merge(baseui, repo, rev=revision)
 
     @reraise_safe_exceptions
     def commit(self, wire, message, username):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig('ui', 'username', username)
         commands.commit(baseui, repo, message=message)
 
     @reraise_safe_exceptions
     def rebase(self, wire, source=None, dest=None, abort=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig('ui', 'merge', 'internal:dump')
         rebase.rebase(
             baseui, repo, base=source, dest=dest, abort=abort, keep=not abort)
 
     @reraise_safe_exceptions
     def bookmark(self, wire, bookmark, revision=None):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         commands.bookmark(baseui, repo, bookmark, rev=revision, force=True)