introduce store classes

move store walking from streamclone.py into store.py

mercurial/localrepo.py

@@ -60,30 +60,13 b' class localrepository(repo.repository):'
             if r not in self.supported:
                 raise repo.RepoError(_("requirement '%s' not supported") % r)
 
-        # setup store
-        if "store" in requirements:
-            self.encodefn = store.encodefilename
-            self.decodefn = store.decodefilename
-            self.spath = os.path.join(self.path, "store")
-        else:
-            self.encodefn = lambda x: x
-            self.decodefn = lambda x: x
-            self.spath = self.path
+        self.store = store.store(requirements, self.path)
 
-        try:
-            # files in .hg/ will be created using this mode
-            mode = os.stat(self.spath).st_mode
-            # avoid some useless chmods
-            if (0777 & ~util._umask) == (0777 & mode):
-                mode = None
-        except OSError:
-            mode = None
-
-        self._createmode = mode
-        self.opener.createmode = mode
-        sopener = util.opener(self.spath)
-        sopener.createmode = mode
-        self.sopener = store.encodedopener(sopener, self.encodefn)
+        self.spath = self.store.path
+        self.sopener = self.store.opener
+        self.sjoin = self.store.join
+        self._createmode = self.store.createmode
+        self.opener.createmode = self.store.createmode
 
         self.ui = ui.ui(parentui=parentui)
         try:
@@ -481,10 +464,6 b' class localrepository(repo.repository):'
     def join(self, f):
         return os.path.join(self.path, f)
 
-    def sjoin(self, f):
-        f = self.encodefn(f)
-        return os.path.join(self.spath, f)
-
     def wjoin(self, f):
         return os.path.join(self.root, f)
 
@@ -2061,6 +2040,25 b' class localrepository(repo.repository):'
             return self.stream_in(remote)
         return self.pull(remote, heads)
 
+    def storefiles(self):
+        '''get all *.i and *.d files in the store
+
+        Returns (list of (filename, size), total_bytes)'''
+
+        lock = None
+        try:
+            self.ui.debug('scanning\n')
+            entries = []
+            total_bytes = 0
+            # get consistent snapshot of repo, lock during scan
+            lock = self.lock()
+            for name, size in self.store.walk():
+                entries.append((name, size))
+                total_bytes += size
+            return entries, total_bytes
+        finally:
+            del lock
+
 # used to avoid circular references so destructors work
 def aftertrans(files):
     renamefiles = [tuple(t) for t in files]

mercurial/statichttprepo.py

@@ -55,14 +55,13 b' class statichttprepository(localrepo.loc'
 
         # setup store
         if "store" in requirements:
-            self.encodefn = store.encodefilename
-            self.decodefn = store.decodefilename
             self.spath = self.path + "/store"
         else:
-            self.encodefn = lambda x: x
-            self.decodefn = lambda x: x
             self.spath = self.path
-        self.sopener = store.encodedopener(opener(self.spath), self.encodefn)
+        self.encodefn = store.encodefn(requirements)
+        so = opener(self.spath)
+        self.sopener = lambda path, *args, **kw: so(
+            self.encodefn(path), *args, **kw)
 
         self.manifest = manifest.manifest(self.sopener)
         self.changelog = changelog.changelog(self.sopener)

mercurial/store.py

@@ -5,6 +5,8 b''
 # This software may be used and distributed according to the terms
 # of the GNU General Public License, incorporated herein by reference.
 
+import os, stat, osutil, util
+
 def _buildencodefun():
     e = '_'
     win_reserved = [ord(x) for x in '\\:*?"<>|']
@@ -33,7 +35,91 b' def _buildencodefun():'
 
 encodefilename, decodefilename = _buildencodefun()
 
-def encodedopener(openerfn, fn):
-    def o(path, *args, **kw):
-        return openerfn(fn(path), *args, **kw)
-    return o
+def _dirwalk(path, recurse):
+    '''yields (filename, size)'''
+    for e, kind, st in osutil.listdir(path, stat=True):
+        pe = os.path.join(path, e)
+        if kind == stat.S_IFDIR:
+            if recurse:
+                for x in _dirwalk(pe, True):
+                    yield x
+        elif kind == stat.S_IFREG:
+            yield pe, st.st_size
+
+class _store:
+    '''base class for local repository stores'''
+    def __init__(self, path):
+        self.path = path
+        try:
+            # files in .hg/ will be created using this mode
+            mode = os.stat(self.path).st_mode
+            # avoid some useless chmods
+            if (0777 & ~util._umask) == (0777 & mode):
+                mode = None
+        except OSError:
+            mode = None
+        self.createmode = mode
+
+    def join(self, f):
+        return os.path.join(self.path, f)
+
+    def _revlogfiles(self, relpath='', recurse=False):
+        '''yields (filename, size)'''
+        if relpath:
+            path = os.path.join(self.path, relpath)
+        else:
+            path = self.path
+        striplen = len(self.path) + len(os.sep)
+        filetypes = ('.d', '.i')
+        for f, size in _dirwalk(path, recurse):
+            if (len(f) > 2) and f[-2:] in filetypes:
+                yield util.pconvert(f[striplen:]), size
+
+    def _datafiles(self):
+        for x in self._revlogfiles('data', True):
+            yield x
+
+    def walk(self):
+        '''yields (direncoded filename, size)'''
+        # yield data files first
+        for x in self._datafiles():
+            yield x
+        # yield manifest before changelog
+        meta = util.sort(self._revlogfiles())
+        meta.reverse()
+        for x in meta:
+            yield x
+
+class directstore(_store):
+    def __init__(self, path):
+        _store.__init__(self, path)
+        self.encodefn = lambda x: x
+        self.opener = util.opener(self.path)
+        self.opener.createmode = self.createmode
+
+class encodedstore(_store):
+    def __init__(self, path):
+        _store.__init__(self, os.path.join(path, 'store'))
+        self.encodefn = encodefilename
+        op = util.opener(self.path)
+        op.createmode = self.createmode
+        self.opener = lambda f, *args, **kw: op(self.encodefn(f), *args, **kw)
+
+    def _datafiles(self):
+        for f, size in self._revlogfiles('data', True):
+            yield decodefilename(f), size
+
+    def join(self, f):
+        return os.path.join(self.path, self.encodefn(f))
+
+def encodefn(requirements):
+    if 'store' not in requirements:
+        return lambda x: x
+    else:
+        return encodefilename
+
+def store(requirements, path):
+    if 'store' not in requirements:
+        return directstore(path)
+    else:
+        return encodedstore(path)

mercurial/streamclone.py

@@ -5,40 +5,12 b''
 # This software may be used and distributed according to the terms
 # of the GNU General Public License, incorporated herein by reference.
 
-import os, osutil, stat, util, lock
+import util, lock
 
 # if server supports streaming clone, it advertises "stream"
 # capability with value that is version+flags of repo it is serving.
 # client only streams if it can read that repo format.
 
-def walkrepo(root):
-    '''iterate over metadata files in repository.
-    walk in natural (sorted) order.
-    yields 2-tuples: name of .d or .i file, size of file.'''
-
-    strip_count = len(root) + len(os.sep)
-    def walk(path, recurse):
-        for e, kind, st in osutil.listdir(path, stat=True):
-            pe = os.path.join(path, e)
-            if kind == stat.S_IFDIR:
-                if recurse:
-                    for x in walk(pe, True):
-                        yield x
-            else:
-                if kind != stat.S_IFREG or len(e) < 2:
-                    continue
-                sfx = e[-2:]
-                if sfx in ('.d', '.i'):
-                    yield pe[strip_count:], st.st_size
-    # write file data first
-    for x in walk(os.path.join(root, 'data'), True):
-        yield x
-    # write manifest before changelog
-    meta = util.sort(walk(root, False))
-    meta.reverse()
-    for x in meta:
-        yield x
-
 # stream file format is simple.
 #
 # server writes out line that says how many files, how many total
@@ -59,28 +31,14 b' def stream_out(repo, fileobj, untrusted='
         fileobj.write('1\n')
         return
 
-    # get consistent snapshot of repo. lock during scan so lock not
-    # needed while we stream, and commits can happen.
-    repolock = None
     try:
-        try:
-            repolock = repo.lock()
-        except (lock.LockHeld, lock.LockUnavailable), inst:
-            repo.ui.warn('locking the repository failed: %s\n' % (inst,))
-            fileobj.write('2\n')
-            return
+        entries, total_bytes = repo.storefiles()
+    except (lock.LockHeld, lock.LockUnavailable), inst:
+        repo.ui.warn('locking the repository failed: %s\n' % (inst,))
+        fileobj.write('2\n')
+        return
 
-        fileobj.write('0\n')
-        repo.ui.debug('scanning\n')
-        entries = []
-        total_bytes = 0
-        for name, size in walkrepo(repo.spath):
-            name = repo.decodefn(util.pconvert(name))
-            entries.append((name, size))
-            total_bytes += size
-    finally:
-        del repolock
-
+    fileobj.write('0\n')
     repo.ui.debug('%d files, %d bytes to transfer\n' %
                   (len(entries), total_bytes))
     fileobj.write('%d %d\n' % (len(entries), total_bytes))