git: use a fetch_sync based creation of remote repos.

- skips fragile dulwich code that has problems with large repos
- is in sync with logic that does remote fetch
- is faster and more efficient

rhodecode/api/tests/test_create_pull_request.py

@@ -66,6 +66,7 b' class TestCreatePullRequestApi(object):'
         expected_message = "Created new pull request `{title}`".format(
             title=data['title'])
         result = response.json
+        assert result['error'] == None
         assert result['result']['msg'] == expected_message
         pull_request_id = result['result']['pull_request_id']
         pull_request = PullRequestModel().get(pull_request_id)
@@ -88,6 +89,7 b' class TestCreatePullRequestApi(object):'
         expected_message = "Created new pull request `{title}`".format(
             title=data['title'])
         result = response.json
+        assert result['error'] == None
         assert result['result']['msg'] == expected_message
         pull_request_id = result['result']['pull_request_id']
         pull_request = PullRequestModel().get(pull_request_id)
@@ -127,6 +129,7 b' class TestCreatePullRequestApi(object):'
         expected_message = "Created new pull request `{title}`".format(
             title=data['title'])
         result = response.json
+        assert result['error'] == None
         assert result['result']['msg'] == expected_message
         pull_request_id = result['result']['pull_request_id']
         pull_request = PullRequestModel().get(pull_request_id)
@@ -170,6 +173,7 b' class TestCreatePullRequestApi(object):'
         expected_message = "Created new pull request `{title}`".format(
             title=data['title'])
         result = response.json
+        assert result['error'] == None
         assert result['result']['msg'] == expected_message
         pull_request_id = result['result']['pull_request_id']
         pull_request = PullRequestModel().get(pull_request_id)

rhodecode/lib/celerylib/tasks.py

@@ -123,7 +123,7 b' def create_repo(form_data, cur_user):'
         'enable_downloads', defs.get('repo_enable_downloads'))
 
     try:
-        repo = RepoModel()._create_repo(
+        RepoModel()._create_repo(
             repo_name=repo_name_full,
             repo_type=repo_type,
             description=description,

rhodecode/lib/vcs/backends/git/repository.py

@@ -58,13 +58,13 b' class GitRepository(BaseRepository):'
     contact = BaseRepository.DEFAULT_CONTACT
 
     def __init__(self, repo_path, config=None, create=False, src_url=None,
-                 update_after_clone=False, with_wire=None, bare=False):
+                 do_workspace_checkout=False, with_wire=None, bare=False):
 
         self.path = safe_str(os.path.abspath(repo_path))
         self.config = config if config else self.get_default_config()
         self.with_wire = with_wire
 
-        self._init_repo(create, src_url, update_after_clone, bare)
+        self._init_repo(create, src_url, do_workspace_checkout, bare)
 
         # caches
         self._commit_ids = {}
@@ -145,24 +145,36 b' class GitRepository(BaseRepository):'
             pass
         return False
 
-    def _init_repo(self, create, src_url=None, update_after_clone=False,
+    def _init_repo(self, create, src_url=None, do_workspace_checkout=False,
                    bare=False):
         if create and os.path.exists(self.path):
             raise RepositoryError(
                 "Cannot create repository at %s, location already exist"
                 % self.path)
 
+        if bare and do_workspace_checkout:
+            raise RepositoryError("Cannot update a bare repository")
         try:
-            if create and src_url:
+
+            if src_url:
+                # check URL before any actions
                 GitRepository.check_url(src_url, self.config)
-                self.clone(src_url, update_after_clone, bare)
-            elif create:
+
+            if create:
                 os.makedirs(self.path, mode=0755)
 
                 if bare:
                     self._remote.init_bare()
                 else:
                     self._remote.init()
+
+                if src_url and bare:
+                    # bare repository only allows a fetch and checkout is not allowed
+                    self.fetch(src_url, commit_ids=None)
+                elif src_url:
+                    self.pull(src_url, commit_ids=None,
+                              update_after=do_workspace_checkout)
+
             else:
                 if not self._remote.assert_correct_path():
                     raise RepositoryError(
@@ -630,49 +642,27 b' class GitRepository(BaseRepository):'
         """
         return GitInMemoryCommit(self)
 
-    def clone(self, url, update_after_clone=True, bare=False):
+    def pull(self, url, commit_ids=None, update_after=False):
         """
-        Tries to clone commits from external location.
-
-        :param update_after_clone: If set to ``False``, git won't checkout
-          working directory
-        :param bare: If set to ``True``, repository would be cloned into
-          *bare* git repository (no working directory at all).
-        """
-        # init_bare and init expect empty dir created to proceed
-        if not os.path.exists(self.path):
-            os.mkdir(self.path)
+        Pull changes from external location. Pull is different in GIT
+        that fetch since it's doing a checkout
 
-        if bare:
-            self._remote.init_bare()
-        else:
-            self._remote.init()
-
-        deferred = '^{}'
-        valid_refs = ('refs/heads', 'refs/tags', 'HEAD')
-
-        return self._remote.clone(
-            url, deferred, valid_refs, update_after_clone)
-
-    def pull(self, url, commit_ids=None):
+        :param commit_ids: Optional. Can be set to a list of commit ids
+           which shall be pulled from the other repository.
         """
-        Tries to pull changes from external location. We use fetch here since
-        pull in get does merges and we want to be compatible with hg backend so
-        pull == fetch in this case
-        """
-        self.fetch(url, commit_ids=commit_ids)
+        refs = None
+        if commit_ids is not None:
+            remote_refs = self._remote.get_remote_refs(url)
+            refs = [ref for ref in remote_refs if remote_refs[ref] in commit_ids]
+        self._remote.pull(url, refs=refs, update_after=update_after)
+        self._remote.invalidate_vcs_cache()
 
     def fetch(self, url, commit_ids=None):
         """
-        Tries to fetch changes from external location.
+        Fetch all git objects from external location.
         """
-        refs = None
-
-        if commit_ids is not None:
-            remote_refs = self._remote.get_remote_refs(url)
-            refs = [
-                ref for ref in remote_refs if remote_refs[ref] in commit_ids]
-        self._remote.fetch(url, refs=refs)
+        self._remote.sync_fetch(url, refs=commit_ids)
+        self._remote.invalidate_vcs_cache()
 
     def push(self, url):
         refs = None

rhodecode/lib/vcs/backends/hg/repository.py

@@ -57,7 +57,7 b' class MercurialRepository(BaseRepository'
     DEFAULT_BRANCH_NAME = 'default'
 
     def __init__(self, repo_path, config=None, create=False, src_url=None,
-                 update_after_clone=False, with_wire=None):
+                 do_workspace_checkout=False, with_wire=None, bare=False):
         """
         Raises RepositoryError if repository could not be find at the given
         ``repo_path``.
@@ -67,8 +67,9 b' class MercurialRepository(BaseRepository'
         :param create=False: if set to True, would try to create repository if
            it does not exist rather than raising exception
         :param src_url=None: would try to clone repository from given location
-        :param update_after_clone=False: sets update of working copy after
+        :param do_workspace_checkout=False: sets update of working copy after
            making a clone
+        :param bare: not used, compatible with other VCS
         """
 
         self.path = safe_str(os.path.abspath(repo_path))
@@ -79,7 +80,7 b' class MercurialRepository(BaseRepository'
             default=[('extensions', 'largefiles', '1')])
         self.with_wire = with_wire
 
-        self._init_repo(create, src_url, update_after_clone)
+        self._init_repo(create, src_url, do_workspace_checkout)
 
         # caches
         self._commit_ids = {}
@@ -328,7 +329,7 b' class MercurialRepository(BaseRepository'
     def is_valid_repository(path):
         return os.path.isdir(os.path.join(path, '.hg'))
 
-    def _init_repo(self, create, src_url=None, update_after_clone=False):
+    def _init_repo(self, create, src_url=None, do_workspace_checkout=False):
         """
         Function will check for mercurial repository in given path. If there
         is no repository in that path it will raise an exception unless
@@ -337,7 +338,7 b' class MercurialRepository(BaseRepository'
 
         If `src_url` is given, would try to clone repository from the
         location at given clone_point. Additionally it'll make update to
-        working copy accordingly to `update_after_clone` flag.
+        working copy accordingly to `do_workspace_checkout` flag.
         """
         if create and os.path.exists(self.path):
             raise RepositoryError(
@@ -348,7 +349,7 b' class MercurialRepository(BaseRepository'
             url = str(self._get_url(src_url))
             MercurialRepository.check_url(url, self.config)
 
-            self._remote.clone(url, self.path, update_after_clone)
+            self._remote.clone(url, self.path, do_workspace_checkout)
 
             # Don't try to create if we've already cloned repo
             create = False
@@ -538,7 +539,7 b' class MercurialRepository(BaseRepository'
 
     def pull(self, url, commit_ids=None):
         """
-        Tries to pull changes from external location.
+        Pull changes from external location.
 
         :param commit_ids: Optional. Can be set to a list of commit ids
            which shall be pulled from the other repository.
@@ -547,6 +548,12 b' class MercurialRepository(BaseRepository'
         self._remote.pull(url, commit_ids=commit_ids)
         self._remote.invalidate_vcs_cache()
 
+    def fetch(self, url, commit_ids=None):
+        """
+        Backward compatibility with GIT fetch==pull
+        """
+        return self.pull(url, commit_ids=commit_ids)
+
     def push(self, url):
         url = self._get_url(url)
         self._remote.sync_push(url)

rhodecode/lib/vcs/backends/svn/repository.py

@@ -68,7 +68,7 b' class SubversionRepository(base.BaseRepo'
     contact = base.BaseRepository.DEFAULT_CONTACT
     description = base.BaseRepository.DEFAULT_DESCRIPTION
 
-    def __init__(self, repo_path, config=None, create=False, src_url=None,
+    def __init__(self, repo_path, config=None, create=False, src_url=None, bare=False,
                  **kwargs):
         self.path = safe_str(os.path.abspath(repo_path))
         self.config = config if config else self.get_default_config()

rhodecode/model/scm.py

@@ -399,7 +399,7 b' class ScmModel(BaseModel):'
         repo_name = dbrepo.repo_name
         try:
             # TODO: we need to make sure those operations call proper hooks !
-            repo.pull(remote_uri)
+            repo.fetch(remote_uri)
 
             self.mark_for_invalidation(repo_name)
         except Exception:

rhodecode/tests/plugin.py

@@ -581,7 +581,7 b' class Backend(object):'
 
     def create_repo(
             self, commits=None, number_of_commits=0, heads=None,
-            name_suffix=u'', **kwargs):
+            name_suffix=u'', bare=False, **kwargs):
         """
         Create a repository and record it for later cleanup.
 
@@ -591,16 +591,17 b' class Backend(object):'
            commits will be added to the new repository.
         :param heads: Optional. Can be set to a sequence of of commit
            names which shall be pulled in from the master repository.
-
+        :param name_suffix: adds special suffix to generated repo name
+        :param bare: set a repo as bare (no checkout)
         """
         self.repo_name = self._next_repo_name() + name_suffix
         repo = self._fixture.create_repo(
-            self.repo_name, repo_type=self.alias, **kwargs)
+            self.repo_name, repo_type=self.alias, bare=bare, **kwargs)
         self._cleanup_repos.append(repo.repo_name)
 
         commits = commits or [
             {'message': 'Commit %s of %s' % (x, self.repo_name)}
-            for x in xrange(number_of_commits)]
+            for x in range(number_of_commits)]
         self._add_commits_to_repo(repo.scm_instance(), commits)
         if heads:
             self.pull_heads(repo, heads)
@@ -773,14 +774,15 b' class VcsBackend(object):'
         """
         return get_backend(self.alias)
 
-    def create_repo(self, commits=None, number_of_commits=0, _clone_repo=None):
+    def create_repo(self, commits=None, number_of_commits=0, _clone_repo=None,
+                    bare=False):
         repo_name = self._next_repo_name()
         self._repo_path = get_new_dir(repo_name)
         repo_class = get_backend(self.alias)
         src_url = None
         if _clone_repo:
             src_url = _clone_repo.path
-        repo = repo_class(self._repo_path, create=True, src_url=src_url)
+        repo = repo_class(self._repo_path, create=True, src_url=src_url, bare=bare)
         self._cleanup_repos.append(repo)
 
         commits = commits or [
@@ -1158,13 +1160,13 b' class UserUtility(object):'
         return repo_group
 
     def create_repo(self, owner=TEST_USER_ADMIN_LOGIN, parent=None,
-                    auto_cleanup=True, repo_type='hg'):
+                    auto_cleanup=True, repo_type='hg', bare=False):
         repo_name = "{prefix}_repository_{count}".format(
             prefix=self._test_name,
             count=len(self.repos_ids))
 
         repository = self.fixture.create_repo(
-            repo_name, cur_user=owner, repo_group=parent, repo_type=repo_type)
+            repo_name, cur_user=owner, repo_group=parent, repo_type=repo_type, bare=bare)
         if auto_cleanup:
             self.repos_ids.append(repository.repo_id)
         return repository

rhodecode/tests/vcs/test_git.py

@@ -94,7 +94,7 b' class TestGitRepository:'
         repo = GitRepository(TEST_GIT_REPO)
         repo_clone = GitRepository(
             TEST_GIT_REPO_CLONE,
-            src_url=TEST_GIT_REPO, create=True, update_after_clone=True)
+            src_url=TEST_GIT_REPO, create=True, do_workspace_checkout=True)
         assert len(repo.commit_ids) == len(repo_clone.commit_ids)
         # Checking hashes of commits should be enough
         for commit in repo.get_commits():
@@ -111,7 +111,7 b' class TestGitRepository:'
         clone_path = TEST_GIT_REPO_CLONE + '_with_update'
         repo_clone = GitRepository(
             clone_path,
-            create=True, src_url=TEST_GIT_REPO, update_after_clone=True)
+            create=True, src_url=TEST_GIT_REPO, do_workspace_checkout=True)
         assert len(repo.commit_ids) == len(repo_clone.commit_ids)
 
         # check if current workdir was updated
@@ -123,7 +123,7 b' class TestGitRepository:'
         clone_path = TEST_GIT_REPO_CLONE + '_without_update'
         repo_clone = GitRepository(
             clone_path,
-            create=True, src_url=TEST_GIT_REPO, update_after_clone=False)
+            create=True, src_url=TEST_GIT_REPO, do_workspace_checkout=False)
         assert len(repo.commit_ids) == len(repo_clone.commit_ids)
         # check if current workdir was *NOT* updated
         fpath = os.path.join(clone_path, 'MANIFEST.in')
@@ -153,7 +153,7 b' class TestGitRepository:'
         # Note: This is a git specific part of the API, it's only implemented
         # by the git backend.
         source_repo = vcsbackend_git.repo
-        target_repo = vcsbackend_git.create_repo()
+        target_repo = vcsbackend_git.create_repo(bare=True)
         target_repo.fetch(source_repo.path)
         # Note: Get a fresh instance, avoids caching trouble
         target_repo = vcsbackend_git.backend(target_repo.path)
@@ -162,32 +162,31 b' class TestGitRepository:'
     def test_commit_ids(self):
         # there are 112 commits (by now)
         # so we can assume they would be available from now on
-        subset = set([
-            'c1214f7e79e02fc37156ff215cd71275450cffc3',
-            '38b5fe81f109cb111f549bfe9bb6b267e10bc557',
-            'fa6600f6848800641328adbf7811fd2372c02ab2',
-            '102607b09cdd60e2793929c4f90478be29f85a17',
-            '49d3fd156b6f7db46313fac355dca1a0b94a0017',
-            '2d1028c054665b962fa3d307adfc923ddd528038',
-            'd7e0d30fbcae12c90680eb095a4f5f02505ce501',
-            'ff7ca51e58c505fec0dd2491de52c622bb7a806b',
-            'dd80b0f6cf5052f17cc738c2951c4f2070200d7f',
-            '8430a588b43b5d6da365400117c89400326e7992',
-            'd955cd312c17b02143c04fa1099a352b04368118',
-            'f67b87e5c629c2ee0ba58f85197e423ff28d735b',
-            'add63e382e4aabc9e1afdc4bdc24506c269b7618',
-            'f298fe1189f1b69779a4423f40b48edf92a703fc',
-            'bd9b619eb41994cac43d67cf4ccc8399c1125808',
-            '6e125e7c890379446e98980d8ed60fba87d0f6d1',
-            'd4a54db9f745dfeba6933bf5b1e79e15d0af20bd',
-            '0b05e4ed56c802098dfc813cbe779b2f49e92500',
-            '191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e',
-            '45223f8f114c64bf4d6f853e3c35a369a6305520',
-            'ca1eb7957a54bce53b12d1a51b13452f95bc7c7e',
-            'f5ea29fc42ef67a2a5a7aecff10e1566699acd68',
-            '27d48942240f5b91dfda77accd2caac94708cc7d',
-            '622f0eb0bafd619d2560c26f80f09e3b0b0d78af',
-            'e686b958768ee96af8029fe19c6050b1a8dd3b2b'])
+        subset = {'c1214f7e79e02fc37156ff215cd71275450cffc3',
+                  '38b5fe81f109cb111f549bfe9bb6b267e10bc557',
+                  'fa6600f6848800641328adbf7811fd2372c02ab2',
+                  '102607b09cdd60e2793929c4f90478be29f85a17',
+                  '49d3fd156b6f7db46313fac355dca1a0b94a0017',
+                  '2d1028c054665b962fa3d307adfc923ddd528038',
+                  'd7e0d30fbcae12c90680eb095a4f5f02505ce501',
+                  'ff7ca51e58c505fec0dd2491de52c622bb7a806b',
+                  'dd80b0f6cf5052f17cc738c2951c4f2070200d7f',
+                  '8430a588b43b5d6da365400117c89400326e7992',
+                  'd955cd312c17b02143c04fa1099a352b04368118',
+                  'f67b87e5c629c2ee0ba58f85197e423ff28d735b',
+                  'add63e382e4aabc9e1afdc4bdc24506c269b7618',
+                  'f298fe1189f1b69779a4423f40b48edf92a703fc',
+                  'bd9b619eb41994cac43d67cf4ccc8399c1125808',
+                  '6e125e7c890379446e98980d8ed60fba87d0f6d1',
+                  'd4a54db9f745dfeba6933bf5b1e79e15d0af20bd',
+                  '0b05e4ed56c802098dfc813cbe779b2f49e92500',
+                  '191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e',
+                  '45223f8f114c64bf4d6f853e3c35a369a6305520',
+                  'ca1eb7957a54bce53b12d1a51b13452f95bc7c7e',
+                  'f5ea29fc42ef67a2a5a7aecff10e1566699acd68',
+                  '27d48942240f5b91dfda77accd2caac94708cc7d',
+                  '622f0eb0bafd619d2560c26f80f09e3b0b0d78af',
+                  'e686b958768ee96af8029fe19c6050b1a8dd3b2b'}
         assert subset.issubset(set(self.repo.commit_ids))
 
     def test_slicing(self):
@@ -281,12 +280,12 b' TODO: To be written...'
 
         new_branch = 'new_branch'
         assert repo_clone._current_branch() == 'master'
-        assert set(repo_clone.branches) == set(('master',))
+        assert set(repo_clone.branches) == {'master'}
         repo_clone._checkout(new_branch, create=True)
 
         # Branches is a lazy property so we need to recrete the Repo object.
         repo_clone = GitRepository(repo_clone.path)
-        assert set(repo_clone.branches) == set(('master', new_branch))
+        assert set(repo_clone.branches) == {'master', new_branch}
         assert repo_clone._current_branch() == new_branch
 
     def test_checkout(self):
@@ -1171,7 +1170,7 b' class TestGitRegression(BackendTestMixin'
         assert paths == expected_paths
 
 
-class TestDiscoverGitVersion:
+class TestDiscoverGitVersion(object):
 
     def test_returns_git_version(self, baseapp):
         version = discover_git_version()

rhodecode/tests/vcs/test_hg.py

@@ -122,7 +122,7 b' class TestMercurialRepository:'
         repo = MercurialRepository(TEST_HG_REPO)
         repo_clone = MercurialRepository(
             TEST_HG_REPO_CLONE + '_w_update',
-            src_url=TEST_HG_REPO, update_after_clone=True)
+            src_url=TEST_HG_REPO, do_workspace_checkout=True)
         assert len(repo.commit_ids) == len(repo_clone.commit_ids)
 
         # check if current workdir was updated
@@ -133,7 +133,7 b' class TestMercurialRepository:'
         repo = MercurialRepository(TEST_HG_REPO)
         repo_clone = MercurialRepository(
             TEST_HG_REPO_CLONE + '_wo_update',
-            src_url=TEST_HG_REPO, update_after_clone=False)
+            src_url=TEST_HG_REPO, do_workspace_checkout=False)
         assert len(repo.commit_ids) == len(repo_clone.commit_ids)
         assert not os.path.isfile(
             os.path.join(TEST_HG_REPO_CLONE + '_wo_update', 'MANIFEST.in'))