dirstate.read: make 15% faster.

time to read dirstate for my kernel repo goes from 0.22s to 0.18s.

mercurial/dirstate.py

@@ -1,478 +1,488
 """
 dirstate.py - working directory tracking for mercurial
 
 Copyright 2005 Matt Mackall <[e-mail]>
 
 This software may be used and distributed according to the terms
 of the GNU General Public License, incorporated herein by reference.
 """
 
 import struct, os
 from node import *
 from i18n import gettext as _
 from demandload import *
 demandload(globals(), "time bisect stat util re errno")
 
 class dirstate(object):
     format = ">cllll"
 
     def __init__(self, opener, ui, root):
         self.opener = opener
         self.root = root
         self.dirty = 0
         self.ui = ui
         self.map = None
         self.pl = None
         self.copies = {}
         self.ignorefunc = None
         self.blockignore = False
 
     def wjoin(self, f):
         return os.path.join(self.root, f)
 
     def getcwd(self):
         cwd = os.getcwd()
         if cwd == self.root: return ''
         return cwd[len(self.root) + 1:]
 
     def hgignore(self):
         '''return the contents of .hgignore files as a list of patterns.
 
         the files parsed for patterns include:
         .hgignore in the repository root
         any additional files specified in the [ui] section of ~/.hgrc
 
         trailing white space is dropped.
         the escape character is backslash.
         comments start with #.
         empty lines are skipped.
 
         lines can be of the following formats:
 
         syntax: regexp # defaults following lines to non-rooted regexps
         syntax: glob   # defaults following lines to non-rooted globs
         re:pattern     # non-rooted regular expression
         glob:pattern   # non-rooted glob
         pattern        # pattern of the current default type'''
         syntaxes = {'re': 'relre:', 'regexp': 'relre:', 'glob': 'relglob:'}
         def parselines(fp):
             for line in fp:
                 escape = False
                 for i in xrange(len(line)):
                     if escape: escape = False
                     elif line[i] == '\\': escape = True
                     elif line[i] == '#': break
                 line = line[:i].rstrip()
                 if line: yield line
         repoignore = self.wjoin('.hgignore')
         files = [repoignore]
         files.extend(self.ui.hgignorefiles())
         pats = {}
         for f in files:
             try:
                 pats[f] = []
                 fp = open(f)
                 syntax = 'relre:'
                 for line in parselines(fp):
                     if line.startswith('syntax:'):
                         s = line[7:].strip()
                         try:
                             syntax = syntaxes[s]
                         except KeyError:
                             self.ui.warn(_("%s: ignoring invalid "
                                            "syntax '%s'\n") % (f, s))
                         continue
                     pat = syntax + line
                     for s in syntaxes.values():
                         if line.startswith(s):
                             pat = line
                             break
                     pats[f].append(pat)
             except IOError, inst:
                 if f != repoignore:
                     self.ui.warn(_("skipping unreadable ignore file"
                                    " '%s': %s\n") % (f, inst.strerror))
         return pats
 
     def ignore(self, fn):
         '''default match function used by dirstate and
         localrepository.  this honours the repository .hgignore file
         and any other files specified in the [ui] section of .hgrc.'''
         if self.blockignore:
             return False
         if not self.ignorefunc:
             ignore = self.hgignore()
             allpats = []
             [allpats.extend(patlist) for patlist in ignore.values()]
             if allpats:
                 try:
                     files, self.ignorefunc, anypats = (
                         util.matcher(self.root, inc=allpats, src='.hgignore'))
                 except util.Abort:
                     # Re-raise an exception where the src is the right file
                     for f, patlist in ignore.items():
                         files, self.ignorefunc, anypats = (
                             util.matcher(self.root, inc=patlist, src=f))
             else:
                 self.ignorefunc = util.never
         return self.ignorefunc(fn)
 
     def __del__(self):
         if self.dirty:
             self.write()
 
     def __getitem__(self, key):
         try:
             return self.map[key]
         except TypeError:
             self.lazyread()
             return self[key]
 
     def __contains__(self, key):
         self.lazyread()
         return key in self.map
 
     def parents(self):
         self.lazyread()
         return self.pl
 
     def markdirty(self):
         if not self.dirty:
             self.dirty = 1
 
     def setparents(self, p1, p2=nullid):
         self.lazyread()
         self.markdirty()
         self.pl = p1, p2
 
     def state(self, key):
         try:
             return self[key][0]
         except KeyError:
             return "?"
 
     def lazyread(self):
         if self.map is None:
             self.read()
 
-    def read(self):
-        self.map = {}
-        self.pl = [nullid, nullid]
-        try:
-            st = self.opener("dirstate").read()
-            if not st: return
-        except: return
-
+    def parse(self, st):
         self.pl = [st[:20], st[20: 40]]
 
+        # deref fields so they will be local in loop
+        map = self.map
+        copies = self.copies
+        format = self.format
+        unpack = struct.unpack
+
         pos = 40
-        e_size = struct.calcsize(self.format)
+        e_size = struct.calcsize(format)
+
         while pos < len(st):
             newpos = pos + e_size
-            e = struct.unpack(self.format, st[pos:newpos])
+            e = unpack(format, st[pos:newpos])
             l = e[4]
             pos = newpos
             newpos = pos + l
             f = st[pos:newpos]
             if '\0' in f:
                 f, c = f.split('\0')
-                self.copies[f] = c
-            self.map[f] = e[:4]
+                copies[f] = c
+            map[f] = e[:4]
             pos = newpos
 
+    def read(self):
+        self.map = {}
+        self.pl = [nullid, nullid]
+        try:
+            st = self.opener("dirstate").read()
+            if st:
+                self.parse(st)
+        except IOError, err:
+            if err.errno != errno.ENOENT: raise
+
     def copy(self, source, dest):
         self.lazyread()
         self.markdirty()
         self.copies[dest] = source
 
     def copied(self, file):
         return self.copies.get(file, None)
 
     def update(self, files, state, **kw):
         ''' current states:
         n  normal
         m  needs merging
         r  marked for removal
         a  marked for addition'''
 
         if not files: return
         self.lazyread()
         self.markdirty()
         for f in files:
             if state == "r":
                 self.map[f] = ('r', 0, 0, 0)
             else:
                 s = os.lstat(self.wjoin(f))
                 st_size = kw.get('st_size', s.st_size)
                 st_mtime = kw.get('st_mtime', s.st_mtime)
                 self.map[f] = (state, s.st_mode, st_size, st_mtime)
             if self.copies.has_key(f):
                 del self.copies[f]
 
     def forget(self, files):
         if not files: return
         self.lazyread()
         self.markdirty()
         for f in files:
             try:
                 del self.map[f]
             except KeyError:
                 self.ui.warn(_("not in dirstate: %s!\n") % f)
                 pass
 
     def clear(self):
         self.map = {}
         self.copies = {}
         self.markdirty()
 
     def rebuild(self, parent, files):
         self.clear()
         umask = os.umask(0)
         os.umask(umask)
         for f, mode in files:
             if mode:
                 self.map[f] = ('n', ~umask, -1, 0)
             else:
                 self.map[f] = ('n', ~umask & 0666, -1, 0)
         self.pl = (parent, nullid)
         self.markdirty()
 
     def write(self):
         if not self.dirty:
             return
         st = self.opener("dirstate", "w", atomic=True)
         st.write("".join(self.pl))
         for f, e in self.map.items():
             c = self.copied(f)
             if c:
                 f = f + "\0" + c
             e = struct.pack(self.format, e[0], e[1], e[2], e[3], len(f))
             st.write(e + f)
         self.dirty = 0
 
     def filterfiles(self, files):
         ret = {}
         unknown = []
 
         for x in files:
             if x == '.':
                 return self.map.copy()
             if x not in self.map:
                 unknown.append(x)
             else:
                 ret[x] = self.map[x]
 
         if not unknown:
             return ret
 
         b = self.map.keys()
         b.sort()
         blen = len(b)
 
         for x in unknown:
             bs = bisect.bisect(b, x)
             if bs != 0 and  b[bs-1] == x:
                 ret[x] = self.map[x]
                 continue
             while bs < blen:
                 s = b[bs]
                 if len(s) > len(x) and s.startswith(x) and s[len(x)] == '/':
                     ret[s] = self.map[s]
                 else:
                     break
                 bs += 1
         return ret
 
     def supported_type(self, f, st, verbose=False):
         if stat.S_ISREG(st.st_mode):
             return True
         if verbose:
             kind = 'unknown'
             if stat.S_ISCHR(st.st_mode): kind = _('character device')
             elif stat.S_ISBLK(st.st_mode): kind = _('block device')
             elif stat.S_ISFIFO(st.st_mode): kind = _('fifo')
             elif stat.S_ISLNK(st.st_mode): kind = _('symbolic link')
             elif stat.S_ISSOCK(st.st_mode): kind = _('socket')
             elif stat.S_ISDIR(st.st_mode): kind = _('directory')
             self.ui.warn(_('%s: unsupported file type (type is %s)\n') % (
                 util.pathto(self.getcwd(), f),
                 kind))
         return False
 
     def statwalk(self, files=None, match=util.always, dc=None, ignored=False,
                  badmatch=None):
         self.lazyread()
 
         # walk all files by default
         if not files:
             files = [self.root]
             if not dc:
                 dc = self.map.copy()
         elif not dc:
             dc = self.filterfiles(files)
 
         def statmatch(file_, stat):
             file_ = util.pconvert(file_)
             if not ignored and file_ not in dc and self.ignore(file_):
                 return False
             return match(file_)
 
         return self.walkhelper(files=files, statmatch=statmatch, dc=dc,
                                badmatch=badmatch)
 
     def walk(self, files=None, match=util.always, dc=None, badmatch=None):
         # filter out the stat
         for src, f, st in self.statwalk(files, match, dc, badmatch=badmatch):
             yield src, f
 
     # walk recursively through the directory tree, finding all files
     # matched by the statmatch function
     #
     # results are yielded in a tuple (src, filename, st), where src
     # is one of:
     # 'f' the file was found in the directory tree
     # 'm' the file was only in the dirstate and not in the tree
     # and st is the stat result if the file was found in the directory.
     #
     # dc is an optional arg for the current dirstate.  dc is not modified
     # directly by this function, but might be modified by your statmatch call.
     #
     def walkhelper(self, files, statmatch, dc, badmatch=None):
         # recursion free walker, faster than os.walk.
         def findfiles(s):
             work = [s]
             while work:
                 top = work.pop()
                 names = os.listdir(top)
                 names.sort()
                 # nd is the top of the repository dir tree
                 nd = util.normpath(top[len(self.root) + 1:])
                 if nd == '.':
                     nd = ''
                 else:
                     # do not recurse into a repo contained in this
                     # one. use bisect to find .hg directory so speed
                     # is good on big directory.
                     hg = bisect.bisect_left(names, '.hg')
                     if hg < len(names) and names[hg] == '.hg':
                         if os.path.isdir(os.path.join(top, '.hg')):
                             continue
                 for f in names:
                     np = util.pconvert(os.path.join(nd, f))
                     if seen(np):
                         continue
                     p = os.path.join(top, f)
                     # don't trip over symlinks
                     st = os.lstat(p)
                     if stat.S_ISDIR(st.st_mode):
                         ds = os.path.join(nd, f +'/')
                         if statmatch(ds, st):
                             work.append(p)
                         if statmatch(np, st) and np in dc:
                             yield 'm', np, st
                     elif statmatch(np, st):
                         if self.supported_type(np, st):
                             yield 'f', np, st
                         elif np in dc:
                             yield 'm', np, st
 
         known = {'.hg': 1}
         def seen(fn):
             if fn in known: return True
             known[fn] = 1
 
         # step one, find all files that match our criteria
         files.sort()
         for ff in util.unique(files):
             f = self.wjoin(ff)
             try:
                 st = os.lstat(f)
             except OSError, inst:
                 nf = util.normpath(ff)
                 found = False
                 for fn in dc:
                     if nf == fn or (fn.startswith(nf) and fn[len(nf)] == '/'):
                         found = True
                         break
                 if not found:
                     if inst.errno != errno.ENOENT or not badmatch:
                         self.ui.warn('%s: %s\n' % (
                             util.pathto(self.getcwd(), ff),
                             inst.strerror))
                     elif badmatch and badmatch(ff) and statmatch(ff, None):
                         yield 'b', ff, None
                 continue
             if stat.S_ISDIR(st.st_mode):
                 cmp1 = (lambda x, y: cmp(x[1], y[1]))
                 sorted_ = [ x for x in findfiles(f) ]
                 sorted_.sort(cmp1)
                 for e in sorted_:
                     yield e
             else:
                 ff = util.normpath(ff)
                 if seen(ff):
                     continue
                 self.blockignore = True
                 if statmatch(ff, st):
                     if self.supported_type(ff, st, verbose=True):
                         yield 'f', ff, st
                     elif ff in dc:
                         yield 'm', ff, st
                 self.blockignore = False
 
         # step two run through anything left in the dc hash and yield
         # if we haven't already seen it
         ks = dc.keys()
         ks.sort()
         for k in ks:
             if not seen(k) and (statmatch(k, None)):
                 yield 'm', k, None
 
     def changes(self, files=None, match=util.always, show_ignored=None):
         lookup, modified, added, unknown, ignored = [], [], [], [], []
         removed, deleted = [], []
 
         for src, fn, st in self.statwalk(files, match, ignored=show_ignored):
             try:
                 type_, mode, size, time = self[fn]
             except KeyError:
                 if show_ignored and self.ignore(fn):
                     ignored.append(fn)
                 else:
                     unknown.append(fn)
                 continue
             if src == 'm':
                 nonexistent = True
                 if not st:
                     try:
                         f = self.wjoin(fn)
                         st = os.lstat(f)
                     except OSError, inst:
                         if inst.errno != errno.ENOENT:
                             raise
                         st = None
                     # We need to re-check that it is a valid file
                     if st and self.supported_type(fn, st):
                         nonexistent = False
                 # XXX: what to do with file no longer present in the fs
                 # who are not removed in the dirstate ?
                 if nonexistent and type_ in "nm":
                     deleted.append(fn)
                     continue
             # check the common case first
             if type_ == 'n':
                 if not st:
                     st = os.stat(fn)
                 if size >= 0 and (size != st.st_size
                                   or (mode ^ st.st_mode) & 0100):
                     modified.append(fn)
                 elif time != st.st_mtime:
                     lookup.append(fn)
             elif type_ == 'm':
                 modified.append(fn)
             elif type_ == 'a':
                 added.append(fn)
             elif type_ == 'r':
                 removed.append(fn)
 
         return (lookup, modified, added, removed, deleted, unknown, ignored)