shadow-repos: use safer way to destroy shadow repositories.

we had reported errors on removal of shadow repos. Not reproduced, however
suspecting are filesystem sync/symlink race-conditions on shared storage.

End result were existing shadow-repo directories that tricked rhodecode
into thinking shadow repos is existing, but infact it was a dummy
structure semi-removed. Using shutil.move we ENSURE rhodecode doesn't
read those back even if removal fails.

rhodecode/lib/vcs/backends/base.py

@@ -31,6 +31,7 b' import os'
 import re
 import time
 import warnings
+import shutil
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
@@ -521,6 +522,9 b' class BaseRepository(object):'
         """
         raise NotImplementedError
 
+    def _get_shadow_repository_path(self, workspace_id):
+        raise NotImplementedError
+
     def cleanup_merge_workspace(self, workspace_id):
         """
         Remove merge workspace.
@@ -530,7 +534,23 b' class BaseRepository(object):'
 
         :param workspace_id: `workspace_id` unique identifier.
         """
-        raise NotImplementedError
+        shadow_repository_path = self._get_shadow_repository_path(workspace_id)
+        shadow_repository_path_del = '{}.{}.delete'.format(
+            shadow_repository_path, time.time())
+
+        # move the shadow repo, so it never conflicts with the one used.
+        # we use this method because shutil.rmtree had some edge case problems
+        # removing symlinked repositories
+        if not os.path.isdir(shadow_repository_path):
+            return
+
+        shutil.move(shadow_repository_path, shadow_repository_path_del)
+        try:
+            shutil.rmtree(shadow_repository_path_del, ignore_errors=False)
+        except Exception:
+            log.exception('Failed to gracefully remove shadow repo under %s',
+                          shadow_repository_path_del)
+            shutil.rmtree(shadow_repository_path_del, ignore_errors=True)
 
     # ========== #
     # COMMIT API #

rhodecode/lib/vcs/backends/git/repository.py

@@ -25,7 +25,6 b' GIT repository module'
 import logging
 import os
 import re
-import shutil
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
@@ -983,7 +982,3 b' class GitRepository(BaseRepository):'
                 shadow_repository_path, target_ref.name, source_ref.name)
 
         return shadow_repository_path
-
-    def cleanup_merge_workspace(self, workspace_id):
-        shadow_repository_path = self._get_shadow_repository_path(workspace_id)
-        shutil.rmtree(shadow_repository_path, ignore_errors=True)

rhodecode/lib/vcs/backends/hg/repository.py

@@ -24,7 +24,6 b' HG repository module'
 import os
 import logging
 import binascii
-import shutil
 import urllib
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
@@ -698,10 +697,6 b' class MercurialRepository(BaseRepository'
 
         return shadow_repository_path
 
-    def cleanup_merge_workspace(self, workspace_id):
-        shadow_repository_path = self._get_shadow_repository_path(workspace_id)
-        shutil.rmtree(shadow_repository_path, ignore_errors=True)
-
     def _merge_repo(self, shadow_repository_path, target_ref,
                     source_repo, source_ref, merge_message,
                     merger_name, merger_email, dry_run=False,