Merge crew and main.

mercurial/worker.py

@@ -0,0 +1,123 b''
+# worker.py - master-slave parallelism support
+#
+# Copyright 2013 Facebook, Inc.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from i18n import _
+import os, signal, sys, util
+
+def countcpus():
+    '''try to count the number of CPUs on the system'''
+
+    # posix
+    try:
+        n = int(os.sysconf('SC_NPROCESSORS_ONLN'))
+        if n > 0:
+            return n
+    except (AttributeError, ValueError):
+        pass
+
+    # windows
+    try:
+        n = int(os.environ['NUMBER_OF_PROCESSORS'])
+        if n > 0:
+            return n
+    except (KeyError, ValueError):
+        pass
+
+    return 1
+
+def _numworkers(ui):
+    s = ui.config('worker', 'numcpus')
+    if s:
+        try:
+            n = int(s)
+            if n >= 1:
+                return n
+        except ValueError:
+            raise util.Abort(_('number of cpus must be an integer'))
+    return min(max(countcpus(), 4), 32)
+
+if os.name == 'posix':
+    _startupcost = 0.01
+else:
+    _startupcost = 1e30
+
+def worthwhile(ui, costperop, nops):
+    '''try to determine whether the benefit of multiple processes can
+    outweigh the cost of starting them'''
+    linear = costperop * nops
+    workers = _numworkers(ui)
+    benefit = linear - (_startupcost * workers + linear / workers)
+    return benefit >= 0.15
+
+def worker(ui, costperarg, func, staticargs, args):
+    '''run a function, possibly in parallel in multiple worker
+    processes.
+
+    returns a progress iterator
+
+    costperarg - cost of a single task
+
+    func - function to run
+
+    staticargs - arguments to pass to every invocation of the function
+
+    args - arguments to split into chunks, to pass to individual
+    workers
+    '''
+    if worthwhile(ui, costperarg, len(args)):
+        return _platformworker(ui, func, staticargs, args)
+    return func(*staticargs + (args,))
+
+def _posixworker(ui, func, staticargs, args):
+    rfd, wfd = os.pipe()
+    workers = _numworkers(ui)
+    for pargs in partition(args, workers):
+        pid = os.fork()
+        if pid == 0:
+            try:
+                os.close(rfd)
+                for i, item in func(*(staticargs + (pargs,))):
+                    os.write(wfd, '%d %s\n' % (i, item))
+                os._exit(0)
+            except KeyboardInterrupt:
+                os._exit(255)
+    os.close(wfd)
+    fp = os.fdopen(rfd, 'rb', 0)
+    oldhandler = signal.getsignal(signal.SIGINT)
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+    def cleanup():
+        # python 2.4 is too dumb for try/yield/finally
+        signal.signal(signal.SIGINT, oldhandler)
+        problems = 0
+        for i in xrange(workers):
+            problems |= os.wait()[1]
+        if problems:
+            sys.exit(1)
+    try:
+        for line in fp:
+            l = line.split(' ', 1)
+            yield int(l[0]), l[1][:-1]
+    except: # re-raises
+        cleanup()
+        raise
+    cleanup()
+
+if os.name != 'nt':
+    _platformworker = _posixworker
+
+def partition(lst, nslices):
+    '''partition a list into N slices of equal size'''
+    n = len(lst)
+    chunk, slop = n / nslices, n % nslices
+    end = 0
+    for i in xrange(nslices):
+        start = end
+        end = start + chunk
+        if slop:
+            end += 1
+            slop -= 1
+        yield lst[start:end]

mercurial/help/config.txt

@@ -1463,3 +1463,15 b' The full set of options is:'
 
 ``templates``
     Where to find the HTML templates. Default is install path.
+
+``worker``
+----------
+
+Parallel master/worker configuration. We currently perform working
+directory updates in parallel on Unix-like systems, which greatly
+helps performance.
+
+``numcpus``
+    Number of CPUs to use for parallel operations. Default is 4 or the
+    number of CPUs on the system, whichever is larger. A zero or
+    negative value is treated as ``use the default``.

mercurial/merge.py

@@ -7,7 +7,7 b''
 
 from node import nullid, nullrev, hex, bin
 from i18n import _
-import error, util, filemerge, copies, subrepo
+import error, util, filemerge, copies, subrepo, worker
 import errno, os, shutil
 
 class mergestate(object):
@@ -342,6 +342,42 b' def manifestmerge(repo, wctx, p2, pa, br'
 def actionkey(a):
     return a[1] == "r" and -1 or 0, a
 
+def getremove(repo, mctx, overwrite, args):
+    """apply usually-non-interactive updates to the working directory
+
+    mctx is the context to be merged into the working copy
+
+    yields tuples for progress updates
+    """
+    verbose = repo.ui.verbose
+    unlink = util.unlinkpath
+    wjoin = repo.wjoin
+    fctx = mctx.filectx
+    wwrite = repo.wwrite
+    audit = repo.wopener.audit
+    i = 0
+    for arg in args:
+        f = arg[0]
+        if arg[1] == 'r':
+            if verbose:
+                repo.ui.note(_("removing %s\n") % f)
+            audit(f)
+            try:
+                unlink(wjoin(f), ignoremissing=True)
+            except OSError, inst:
+                repo.ui.warn(_("update failed to remove %s: %s!\n") %
+                             (f, inst.strerror))
+        else:
+            if verbose:
+                repo.ui.note(_("getting %s\n") % f)
+            wwrite(f, fctx(f).data(), arg[2][0])
+        if i == 100:
+            yield i, f
+            i = 0
+        i += 1
+    if i > 0:
+        yield i, f
+
 def applyupdates(repo, actions, wctx, mctx, actx, overwrite):
     """apply the merge action list to the working directory
 
@@ -393,22 +429,34 b' def applyupdates(repo, actions, wctx, mc'
             util.unlinkpath(repo.wjoin(f))
 
     numupdates = len(actions)
+    workeractions = [a for a in actions if a[1] in 'gr']
+    updated = len([a for a in workeractions if a[1] == 'g'])
+    removed = len([a for a in workeractions if a[1] == 'r'])
+    actions = [a for a in actions if a[1] not in 'gr']
+
+    hgsub = [a[1] for a in workeractions if a[0] == '.hgsubstate']
+    if hgsub and hgsub[0] == 'r':
+        subrepo.submerge(repo, wctx, mctx, wctx, overwrite)
+
+    z = 0
+    prog = worker.worker(repo.ui, 0.001, getremove, (repo, mctx, overwrite),
+                         workeractions)
+    for i, item in prog:
+        z += i
+        repo.ui.progress(_('updating'), z, item=item, total=numupdates,
+                         unit=_('files'))
+
+    if hgsub and hgsub[0] == 'g':
+        subrepo.submerge(repo, wctx, mctx, wctx, overwrite)
+
+    _updating = _('updating')
+    _files = _('files')
+    progress = repo.ui.progress
+
     for i, a in enumerate(actions):
         f, m, args, msg = a
-        repo.ui.progress(_('updating'), i + 1, item=f, total=numupdates,
-                         unit=_('files'))
-        if m == "r": # remove
-            repo.ui.note(_("removing %s\n") % f)
-            audit(f)
-            if f == '.hgsubstate': # subrepo states need updating
-                subrepo.submerge(repo, wctx, mctx, wctx, overwrite)
-            try:
-                util.unlinkpath(repo.wjoin(f), ignoremissing=True)
-            except OSError, inst:
-                repo.ui.warn(_("update failed to remove %s: %s!\n") %
-                             (f, inst.strerror))
-            removed += 1
-        elif m == "m": # merge
+        progress(_updating, z + i + 1, item=f, total=numupdates, unit=_files)
+        if m == "m": # merge
             if fd == '.hgsubstate': # subrepo states need updating
                 subrepo.submerge(repo, wctx, mctx, wctx.ancestor(mctx),
                                  overwrite)
@@ -423,13 +471,6 b' def applyupdates(repo, actions, wctx, mc'
                     updated += 1
                 else:
                     merged += 1
-        elif m == "g": # get
-            flags, = args
-            repo.ui.note(_("getting %s\n") % f)
-            repo.wwrite(f, mctx.filectx(f).data(), flags)
-            updated += 1
-            if f == '.hgsubstate': # subrepo states need updating
-                subrepo.submerge(repo, wctx, mctx, wctx, overwrite)
         elif m == "d": # directory rename
             f2, fd, flags = args
             if f:
@@ -459,7 +500,7 b' def applyupdates(repo, actions, wctx, mc'
             util.setflags(repo.wjoin(f), 'l' in flags, 'x' in flags)
             updated += 1
     ms.commit()
-    repo.ui.progress(_('updating'), None, total=numupdates, unit=_('files'))
+    progress(_updating, None, total=numupdates, unit=_files)
 
     return updated, merged, removed, unresolved
 

tests/test-graft.t

@@ -150,8 +150,8 b' Graft out of order, skipping a merge and'
    branchmerge: True, force: True, partial: False
    ancestor: 4c60f11aa304, local: 6b9e5368ca4e+, remote: 97f8bfe72746
    e: remote is newer -> g
+  getting e
   updating: e 1/1 files (100.00%)
-  getting e
   e
   grafting revision 4
     searching for copies back to rev 1
@@ -161,8 +161,8 b' Graft out of order, skipping a merge and'
    d: remote is newer -> g
    e: versions differ -> m
     preserving e for resolve of e
+  getting d
   updating: d 1/2 files (50.00%)
-  getting d
   updating: e 2/2 files (100.00%)
   picked tool 'internal:merge' for e (binary False symlink False)
   merging e

tests/test-issue522.t

@@ -32,8 +32,8 b' revision.'
    branchmerge: True, force: False, partial: False
    ancestor: bbd179dfa0a7, local: 71766447bdbb+, remote: 4d9e78aaceee
    foo: remote is newer -> g
+  getting foo
   updating: foo 1/1 files (100.00%)
-  getting foo
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
 

tests/test-issue672.t

@@ -36,10 +36,9 b' http://mercurial.selenic.com/bts/issue67'
    ancestor: 81f4b099af3d, local: c64f439569a9+, remote: c12dcd37c90a
    1: other deleted -> r
    1a: remote created -> g
-  updating: 1 1/2 files (50.00%)
   removing 1
+  getting 1a
   updating: 1a 2/2 files (100.00%)
-  getting 1a
   1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
 

tests/test-largefiles.t

@@ -1679,8 +1679,8 b' largefiles pulled on update - no server '
    branchmerge: False, force: False, partial: False
    ancestor: 000000000000, local: 000000000000+, remote: cf03e5bb9936
    .hglf/f1: remote created -> g
+  getting .hglf/f1
   updating: .hglf/f1 1/1 files (100.00%)
-  getting .hglf/f1
   getting changed largefiles
   using http://localhost:$HGPORT2/
   sending capabilities command

tests/test-rename-dir-merge.t

@@ -44,16 +44,13 b''
    a/c: remote renamed directory to b/c -> d
    b/a: remote created -> g
    b/b: remote created -> g
-  updating: a/a 1/5 files (20.00%)
   removing a/a
-  updating: a/b 2/5 files (40.00%)
   removing a/b
-  updating: a/c 3/5 files (60.00%)
+  getting b/a
+  getting b/b
+  updating: b/b 4/5 files (80.00%)
+  updating: a/c 5/5 files (100.00%)
   moving a/c to b/c
-  updating: b/a 4/5 files (80.00%)
-  getting b/a
-  updating: b/b 5/5 files (100.00%)
-  getting b/b
   3 files updated, 0 files merged, 2 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
 

tests/test-rename-merge1.t

@@ -41,17 +41,17 b''
    a2: divergent renames -> dr
    b2: remote created -> g
   removing a
-  updating: a 1/3 files (33.33%)
+  getting b2
+  updating: b2 1/3 files (33.33%)
+  updating: a 2/3 files (66.67%)
   picked tool 'internal:merge' for b (binary False symlink False)
   merging a and b to b
   my b@044f8520aeeb+ other b@85c198ef2f6c ancestor a@af1939970a1c
    premerge successful
-  updating: a2 2/3 files (66.67%)
+  updating: a2 3/3 files (100.00%)
   note: possible conflict - a2 was renamed multiple times to:
    c2
    b2
-  updating: b2 3/3 files (100.00%)
-  getting b2
   1 files updated, 1 files merged, 0 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
 
@@ -183,11 +183,11 b' Check for issue3074'
    ancestor: 19d7f95df299, local: 0084274f6b67+, remote: 5d32493049f0
    file: rename and delete -> rd
    newfile: remote created -> g
-  updating: file 1/2 files (50.00%)
+  getting newfile
+  updating: newfile 1/2 files (50.00%)
+  updating: file 2/2 files (100.00%)
   note: possible conflict - file was deleted and renamed to:
    newfile
-  updating: newfile 2/2 files (100.00%)
-  getting newfile
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
   $ hg status

tests/test-rename-merge2.t

@@ -126,8 +126,8 b' args:'
     preserving b for resolve of b
    rev: versions differ -> m
     preserving rev for resolve of rev
+  getting a
   updating: a 1/3 files (33.33%)
-  getting a
   updating: b 2/3 files (66.67%)
   picked tool 'python ../merge' for b (binary False symlink False)
   merging b and a to b
@@ -231,8 +231,8 b' args:'
    b: remote created -> g
    rev: versions differ -> m
     preserving rev for resolve of rev
+  getting b
   updating: b 1/2 files (50.00%)
-  getting b
   updating: rev 2/2 files (100.00%)
   picked tool 'python ../merge' for rev (binary False symlink False)
   merging rev
@@ -289,10 +289,9 b' args:'
    b: remote created -> g
    rev: versions differ -> m
     preserving rev for resolve of rev
-  updating: a 1/3 files (33.33%)
   removing a
+  getting b
   updating: b 2/3 files (66.67%)
-  getting b
   updating: rev 3/3 files (100.00%)
   picked tool 'python ../merge' for rev (binary False symlink False)
   merging rev
@@ -380,12 +379,12 b' m "um a c" "um x c" " " "10 do merg'
    c: remote created -> g
    rev: versions differ -> m
     preserving rev for resolve of rev
-  updating: a 1/3 files (33.33%)
+  getting c
+  updating: c 1/3 files (33.33%)
+  updating: a 2/3 files (66.67%)
   note: possible conflict - a was renamed multiple times to:
    b
    c
-  updating: c 2/3 files (66.67%)
-  getting c
   updating: rev 3/3 files (100.00%)
   picked tool 'python ../merge' for rev (binary False symlink False)
   merging rev
@@ -439,8 +438,8 b' m "um a c" "um x c" " " "10 do merg'
     preserving b for resolve of b
    rev: versions differ -> m
     preserving rev for resolve of rev
+  removing a
   updating: a 1/3 files (33.33%)
-  removing a
   updating: b 2/3 files (66.67%)
   picked tool 'python ../merge' for b (binary False symlink False)
   merging b
@@ -469,8 +468,8 b' m "um a c" "um x c" " " "10 do merg'
     preserving b for resolve of b
    rev: versions differ -> m
     preserving rev for resolve of rev
+  getting a
   updating: a 1/3 files (33.33%)
-  getting a
   updating: b 2/3 files (66.67%)
   picked tool 'python ../merge' for b (binary False symlink False)
   merging b
@@ -500,8 +499,8 b' m "um a c" "um x c" " " "10 do merg'
     preserving b for resolve of b
    rev: versions differ -> m
     preserving rev for resolve of rev
+  removing a
   updating: a 1/3 files (33.33%)
-  removing a
   updating: b 2/3 files (66.67%)
   picked tool 'python ../merge' for b (binary False symlink False)
   merging b
@@ -530,8 +529,8 b' m "um a c" "um x c" " " "10 do merg'
     preserving b for resolve of b
    rev: versions differ -> m
     preserving rev for resolve of rev
+  getting a
   updating: a 1/3 files (33.33%)
-  getting a
   updating: b 2/3 files (66.67%)
   picked tool 'python ../merge' for b (binary False symlink False)
   merging b
@@ -591,8 +590,8 b' m "um a c" "um x c" " " "10 do merg'
     preserving b for resolve of b
    rev: versions differ -> m
     preserving rev for resolve of rev
+  getting a
   updating: a 1/3 files (33.33%)
-  getting a
   updating: b 2/3 files (66.67%)
   picked tool 'python ../merge' for b (binary False symlink False)
   merging b
@@ -731,13 +730,13 b' m "nm a b" "um x a" " " "22 get a, '
    c: remote created -> g
    rev: versions differ -> m
     preserving rev for resolve of rev
-  updating: b 1/3 files (33.33%)
+  getting c
+  updating: c 1/3 files (33.33%)
+  updating: b 2/3 files (66.67%)
   picked tool 'python ../merge' for b (binary False symlink False)
   merging b and a to b
   my b@02963e448370+ other a@2b958612230f ancestor a@924404dff337
    premerge successful
-  updating: c 2/3 files (66.67%)
-  getting c
   updating: rev 3/3 files (100.00%)
   picked tool 'python ../merge' for rev (binary False symlink False)
   merging rev

tests/test-subrepo.t

@@ -214,8 +214,8 b' merge tests'
    branchmerge: False, force: False, partial: False
    ancestor: 60ca1237c194, local: 60ca1237c194+, remote: 6747d179aa9a
    t: remote is newer -> g
+  getting t
   updating: t 1/1 files (100.00%)
-  getting t
   0 files updated, 0 files merged, 0 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
   $ hg debugsub

tests/test-up-local-change.t

@@ -49,12 +49,12 b''
    a: versions differ -> m
     preserving a for resolve of a
    b: remote created -> g
-  updating: a 1/2 files (50.00%)
+  getting b
+  updating: b 1/2 files (50.00%)
+  updating: a 2/2 files (100.00%)
   picked tool 'true' for a (binary False symlink False)
   merging a
   my a@c19d34741b0a+ other a@1e71731e6fbb ancestor a@c19d34741b0a
-  updating: b 2/2 files (100.00%)
-  getting b
   1 files updated, 1 files merged, 0 files removed, 0 files unresolved
   $ hg parents
   changeset:   1:1e71731e6fbb
@@ -70,8 +70,8 b''
    b: other deleted -> r
    a: versions differ -> m
     preserving a for resolve of a
+  removing b
   updating: b 1/2 files (50.00%)
-  removing b
   updating: a 2/2 files (100.00%)
   picked tool 'true' for a (binary False symlink False)
   merging a
@@ -103,12 +103,12 b''
    a: versions differ -> m
     preserving a for resolve of a
    b: remote created -> g
-  updating: a 1/2 files (50.00%)
+  getting b
+  updating: b 1/2 files (50.00%)
+  updating: a 2/2 files (100.00%)
   picked tool 'true' for a (binary False symlink False)
   merging a
   my a@c19d34741b0a+ other a@1e71731e6fbb ancestor a@c19d34741b0a
-  updating: b 2/2 files (100.00%)
-  getting b
   1 files updated, 1 files merged, 0 files removed, 0 files unresolved
   $ hg parents
   changeset:   1:1e71731e6fbb

tests/test-update-reverse.t

@@ -71,12 +71,10 b''
    side1: other deleted -> r
    side2: other deleted -> r
    main: remote created -> g
-  updating: side1 1/3 files (33.33%)
   removing side1
-  updating: side2 2/3 files (66.67%)
   removing side2
+  getting main
   updating: main 3/3 files (100.00%)
-  getting main
   1 files updated, 0 files merged, 2 files removed, 0 files unresolved
 
   $ ls