merge default into stable for 3.9 code freeze

Makefile

@@ -1,261 +1,261
 # If you want to change PREFIX, do not just edit it below. The changed
 # value wont get passed on to recursive make calls. You should instead
 # override the variable on the command like:
 #
 # % make PREFIX=/opt/ install
 
 export PREFIX=/usr/local
 PYTHON=python
 $(eval HGROOT := $(shell pwd))
 HGPYTHONS ?= $(HGROOT)/build/pythons
 PURE=
 PYFILES:=$(shell find mercurial hgext doc -name '*.py')
 DOCFILES=mercurial/help/*.txt
 export LANGUAGE=C
 export LC_ALL=C
 TESTFLAGS ?= $(shell echo $$HGTESTFLAGS)
 
 # Set this to e.g. "mingw32" to use a non-default compiler.
 COMPILER=
 
 COMPILERFLAG_tmp_ =
 COMPILERFLAG_tmp_${COMPILER} ?= -c $(COMPILER)
 COMPILERFLAG=${COMPILERFLAG_tmp_${COMPILER}}
 
 help:
 	@echo 'Commonly used make targets:'
 	@echo '  all          - build program and documentation'
 	@echo '  install      - install program and man pages to $$PREFIX ($(PREFIX))'
 	@echo '  install-home - install with setup.py install --home=$$HOME ($(HOME))'
 	@echo '  local        - build for inplace usage'
 	@echo '  tests        - run all tests in the automatic test suite'
 	@echo '  test-foo     - run only specified tests (e.g. test-merge1.t)'
 	@echo '  dist         - run all tests and create a source tarball in dist/'
 	@echo '  clean        - remove files created by other targets'
 	@echo '                 (except installed files or dist source tarball)'
 	@echo '  update-pot   - update i18n/hg.pot'
 	@echo
 	@echo 'Example for a system-wide installation under /usr/local:'
 	@echo '  make all && su -c "make install" && hg version'
 	@echo
 	@echo 'Example for a local installation (usable in this directory):'
 	@echo '  make local && ./hg version'
 
 all: build doc
 
 local:
 	$(PYTHON) setup.py $(PURE) \
 	  build_py -c -d . \
 	  build_ext $(COMPILERFLAG) -i \
 	  build_hgexe $(COMPILERFLAG) -i \
 	  build_mo
 	env HGRCPATH= $(PYTHON) hg version
 
 build:
 	$(PYTHON) setup.py $(PURE) build $(COMPILERFLAG)
 
 wheel:
 	FORCE_SETUPTOOLS=1 $(PYTHON) setup.py $(PURE) bdist_wheel $(COMPILERFLAG)
 
 doc:
 	$(MAKE) -C doc
 
 clean:
 	-$(PYTHON) setup.py clean --all # ignore errors from this command
 	find contrib doc hgext hgext3rd i18n mercurial tests \
 		\( -name '*.py[cdo]' -o -name '*.so' \) -exec rm -f '{}' ';'
 	rm -f $(addprefix mercurial/,$(notdir $(wildcard mercurial/pure/[a-z]*.py)))
 	rm -f MANIFEST MANIFEST.in hgext/__index__.py tests/*.err
 	rm -f mercurial/__modulepolicy__.py
 	if test -d .hg; then rm -f mercurial/__version__.py; fi
 	rm -rf build packages mercurial/locale
 	$(MAKE) -C doc clean
 	$(MAKE) -C contrib/chg distclean
 
 install: install-bin install-doc
 
 install-bin: build
 	$(PYTHON) setup.py $(PURE) install --root="$(DESTDIR)/" --prefix="$(PREFIX)" --force
 
 install-doc: doc
 	cd doc && $(MAKE) $(MFLAGS) install
 
 install-home: install-home-bin install-home-doc
 
 install-home-bin: build
 	$(PYTHON) setup.py $(PURE) install --home="$(HOME)" --prefix="" --force
 
 install-home-doc: doc
 	cd doc && $(MAKE) $(MFLAGS) PREFIX="$(HOME)" install
 
 MANIFEST-doc:
 	$(MAKE) -C doc MANIFEST
 
 MANIFEST.in: MANIFEST-doc
 	hg manifest | sed -e 's/^/include /' > MANIFEST.in
 	echo include mercurial/__version__.py >> MANIFEST.in
 	sed -e 's/^/include /' < doc/MANIFEST >> MANIFEST.in
 
 dist:	tests dist-notests
 
 dist-notests:	doc MANIFEST.in
 	TAR_OPTIONS="--owner=root --group=root --mode=u+w,go-w,a+rX-s" $(PYTHON) setup.py -q sdist
 
 check: tests
 
 tests:
 	cd tests && $(PYTHON) run-tests.py $(TESTFLAGS)
 
 test-%:
 	cd tests && $(PYTHON) run-tests.py $(TESTFLAGS) $@
 
 testpy-%:
 	@echo Looking for Python $* in $(HGPYTHONS)
 	[ -e $(HGPYTHONS)/$*/bin/python ] || ( \
 	cd $$(mktemp --directory --tmpdir) && \
         $(MAKE) -f $(HGROOT)/contrib/Makefile.python PYTHONVER=$* PREFIX=$(HGPYTHONS)/$* python )
 	cd tests && $(HGPYTHONS)/$*/bin/python run-tests.py $(TESTFLAGS)
 
 check-code:
 	hg manifest | xargs python contrib/check-code.py
 
 update-pot: i18n/hg.pot
 
 i18n/hg.pot: $(PYFILES) $(DOCFILES) i18n/posplit i18n/hggettext
 	$(PYTHON) i18n/hggettext mercurial/commands.py \
 	  hgext/*.py hgext/*/__init__.py \
 	  mercurial/fileset.py mercurial/revset.py \
 	  mercurial/templatefilters.py mercurial/templatekw.py \
 	  mercurial/templater.py \
 	  mercurial/filemerge.py \
 	  mercurial/hgweb/webcommands.py \
 	  $(DOCFILES) > i18n/hg.pot.tmp
         # All strings marked for translation in Mercurial contain
         # ASCII characters only. But some files contain string
         # literals like this '\037\213'. xgettext thinks it has to
         # parse them even though they are not marked for translation.
         # Extracting with an explicit encoding of ISO-8859-1 will make
         # xgettext "parse" and ignore them.
 	echo $(PYFILES) | xargs \
 	  xgettext --package-name "Mercurial" \
 	  --msgid-bugs-address "<[e-mail]>" \
 	  --copyright-holder "Matt Mackall <[e-mail]> and others" \
 	  --from-code ISO-8859-1 --join --sort-by-file --add-comments=i18n: \
 	  -d hg -p i18n -o hg.pot.tmp
 	$(PYTHON) i18n/posplit i18n/hg.pot.tmp
         # The target file is not created before the last step. So it never is in
         # an intermediate state.
 	mv -f i18n/hg.pot.tmp i18n/hg.pot
 
 %.po: i18n/hg.pot
         # work on a temporary copy for never having a half completed target
 	cp $@ $@.tmp
 	msgmerge --no-location --update $@.tmp $^
 	mv -f $@.tmp $@
 
 # Packaging targets
 
 osx:
-	python setup.py install --optimize=1 \
+	/usr/bin/python2.7 setup.py install --optimize=1 \
 	  --root=build/mercurial/ --prefix=/usr/local/ \
 	  --install-lib=/Library/Python/2.7/site-packages/
 	make -C doc all install DESTDIR="$(PWD)/build/mercurial/"
 	mkdir -p $${OUTPUTDIR:-dist}
 	pkgbuild --root build/mercurial/ --identifier org.mercurial-scm.mercurial \
 	  build/mercurial.pkg
 	HGVER=$$((cat build/mercurial/Library/Python/2.7/site-packages/mercurial/__version__.py; echo 'print(version)') | python) && \
 	OSXVER=$$(sw_vers -productVersion | cut -d. -f1,2) && \
 	productbuild --distribution contrib/macosx/distribution.xml \
 	  --package-path build/ \
 	  --version "$${HGVER}" \
 	  --resources contrib/macosx/ \
 	  "$${OUTPUTDIR:-dist/}"/Mercurial-"$${HGVER}"-macosx"$${OSXVER}".pkg
 
 deb:
 	contrib/builddeb
 
 ppa:
 	contrib/builddeb --source-only
 
 docker-debian-jessie:
 	mkdir -p packages/debian-jessie
 	contrib/dockerdeb debian jessie
 
 contrib/docker/ubuntu-%: contrib/docker/ubuntu.template
 	sed "s/__CODENAME__/$*/" $< > $@
 
 docker-ubuntu-trusty: contrib/docker/ubuntu-trusty
 	contrib/dockerdeb ubuntu trusty
 
 docker-ubuntu-trusty-ppa: contrib/docker/ubuntu-trusty
 	contrib/dockerdeb ubuntu trusty --source-only
 
 docker-ubuntu-wily: contrib/docker/ubuntu-wily
 	contrib/dockerdeb ubuntu wily
 
 docker-ubuntu-wily-ppa: contrib/docker/ubuntu-wily
 	contrib/dockerdeb ubuntu wily --source-only
 
 docker-ubuntu-xenial: contrib/docker/ubuntu-xenial
 	contrib/dockerdeb ubuntu xenial
 
 docker-ubuntu-xenial-ppa: contrib/docker/ubuntu-xenial
 	contrib/dockerdeb ubuntu xenial --source-only
 
 fedora20:
 	mkdir -p packages/fedora20
 	contrib/buildrpm
 	cp rpmbuild/RPMS/*/* packages/fedora20
 	cp rpmbuild/SRPMS/* packages/fedora20
 	rm -rf rpmbuild
 
 docker-fedora20:
 	mkdir -p packages/fedora20
 	contrib/dockerrpm fedora20
 
 fedora21:
 	mkdir -p packages/fedora21
 	contrib/buildrpm
 	cp rpmbuild/RPMS/*/* packages/fedora21
 	cp rpmbuild/SRPMS/* packages/fedora21
 	rm -rf rpmbuild
 
 docker-fedora21:
 	mkdir -p packages/fedora21
 	contrib/dockerrpm fedora21
 
 centos5:
 	mkdir -p packages/centos5
 	contrib/buildrpm --withpython
 	cp rpmbuild/RPMS/*/* packages/centos5
 	cp rpmbuild/SRPMS/* packages/centos5
 
 docker-centos5:
 	mkdir -p packages/centos5
 	contrib/dockerrpm centos5 --withpython
 
 centos6:
 	mkdir -p packages/centos6
 	contrib/buildrpm
 	cp rpmbuild/RPMS/*/* packages/centos6
 	cp rpmbuild/SRPMS/* packages/centos6
 
 docker-centos6:
 	mkdir -p packages/centos6
 	contrib/dockerrpm centos6
 
 centos7:
 	mkdir -p packages/centos7
 	contrib/buildrpm
 	cp rpmbuild/RPMS/*/* packages/centos7
 	cp rpmbuild/SRPMS/* packages/centos7
 
 docker-centos7:
 	mkdir -p packages/centos7
 	contrib/dockerrpm centos7
 
 .PHONY: help all local build doc clean install install-bin install-doc \
 	install-home install-home-bin install-home-doc \
 	dist dist-notests check tests check-code update-pot \
 	osx fedora20 docker-fedora20 fedora21 docker-fedora21 \
 	centos5 docker-centos5 centos6 docker-centos6 centos7 docker-centos7

contrib/bash_completion

@@ -1,642 +1,642
 # bash completion for the Mercurial distributed SCM -*- sh -*-
 
 # Docs:
 #
 # If you source this file from your .bashrc, bash should be able to
 # complete a command line that uses hg with all the available commands
 # and options and sometimes even arguments.
 #
 # Mercurial allows you to define additional commands through extensions.
 # Bash should be able to automatically figure out the name of these new
 # commands and their options.  See below for how to define _hg_opt_foo
 # and _hg_cmd_foo functions to fine-tune the completion for option and
 # non-option arguments, respectively.
 #
 #
 # Notes about completion for specific commands:
 #
 # - the completion function for the email command from the patchbomb
 #   extension will try to call _hg_emails to get a list of e-mail
 #   addresses.  It's up to the user to define this function.  For
 #   example, put the addresses of the lists that you usually patchbomb
 #   in ~/.patchbomb-to and the addresses that you usually use to send
 #   the patchbombs in ~/.patchbomb-from and use something like this:
 #
 #      _hg_emails()
 #      {
 #          if [ -r ~/.patchbomb-$1 ]; then
 #              cat ~/.patchbomb-$1
 #          fi
 #      }
 #
 #
 # Writing completion functions for additional commands:
 #
 # If it exists, the function _hg_cmd_foo will be called without
 # arguments to generate the completion candidates for the hg command
 # "foo".  If the command receives some arguments that aren't options
 # even though they start with a "-", you can define a function called
 # _hg_opt_foo to generate the completion candidates.  If _hg_opt_foo
 # doesn't return 0, regular completion for options is attempted.
 #
 # In addition to the regular completion variables provided by bash,
 # the following variables are also set:
 # - $hg - the hg program being used (e.g. /usr/bin/hg)
 # - $cmd - the name of the hg command being completed
 # - $cmd_index - the index of $cmd in $COMP_WORDS
 # - $cur - the current argument being completed
 # - $prev - the argument before $cur
 # - $global_args - "|"-separated list of global options that accept
 #                  an argument (e.g. '--cwd|-R|--repository')
 # - $canonical - 1 if we canonicalized $cmd before calling the function
 #                0 otherwise
 #
 
 shopt -s extglob
 
 _hg_cmd()
 {
     HGPLAIN=1 "$hg" "$@" 2>/dev/null
 }
 
 _hg_commands()
 {
     local commands
     commands="$(HGPLAINEXCEPT=alias _hg_cmd debugcomplete "$cur")" || commands=""
     COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W '$commands' -- "$cur"))
 }
 
 _hg_paths()
 {
     local paths="$(_hg_cmd paths -q)"
     COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W '$paths' -- "$cur"))
 }
 
 _hg_repos()
 {
     local i
     for i in $(compgen -d -- "$cur"); do
         test ! -d "$i"/.hg || COMPREPLY=(${COMPREPLY[@]:-} "$i")
     done
 }
 
 _hg_debugpathcomplete()
 {
     local files="$(_hg_cmd debugpathcomplete $1 "$cur")"
     local IFS=$'\n'
     COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W '$files' -- "$cur"))
 }
 
 _hg_status()
 {
     local files="$(_hg_cmd status -n$1 "glob:$cur**")"
     local IFS=$'\n'
     COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W '$files' -- "$cur"))
 }
 
 _hg_branches()
 {
     local branches="$(_hg_cmd branches -q)"
     local IFS=$'\n'
     COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W '$branches' -- "$cur"))
 }
 
 _hg_bookmarks()
 {
     local bookmarks="$(_hg_cmd bookmarks -q)"
     local IFS=$'\n'
     COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W '$bookmarks' -- "$cur"))
 }
 
 _hg_labels()
 {
     local labels="$(_hg_cmd debugnamecomplete "$cur")"
     local IFS=$'\n'
     COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W '$labels' -- "$cur"))
 }
 
 # this is "kind of" ugly...
 _hg_count_non_option()
 {
     local i count=0
     local filters="$1"
 
     for ((i=1; $i<=$COMP_CWORD; i++)); do
         if [[ "${COMP_WORDS[i]}" != -* ]]; then
             if [[ ${COMP_WORDS[i-1]} == @($filters|$global_args) ]]; then
                 continue
             fi
             count=$(($count + 1))
         fi
     done
 
     echo $(($count - 1))
 }
 
 _hg_fix_wordlist()
 {
     local LASTCHAR=' '
     if [ ${#COMPREPLY[@]} = 1 ]; then
         [ -d "$COMPREPLY" ] && LASTCHAR=/
         COMPREPLY=$(printf %q%s "$COMPREPLY" "$LASTCHAR")
     else
         for ((i=0; i < ${#COMPREPLY[@]}; i++)); do
             [ -d "${COMPREPLY[$i]}" ] && COMPREPLY[$i]=${COMPREPLY[$i]}/
         done
     fi
 }
 
 _hg()
 {
     local cur prev cmd cmd_index opts i aliashg
     # global options that receive an argument
     local global_args='--cwd|-R|--repository'
     local hg="$1"
     local canonical=0
 
     aliashg=$(alias $hg 2>/dev/null)
     if [[ -n "$aliashg" ]]; then
       aliashg=${aliashg#"alias $hg='"}
       aliashg=${aliashg%"'"}
       hg=$aliashg
     fi
 
     COMPREPLY=()
     cur="$2"
     prev="$3"
 
     # searching for the command
     # (first non-option argument that doesn't follow a global option that
     #  receives an argument)
     for ((i=1; $i<=$COMP_CWORD; i++)); do
         if [[ ${COMP_WORDS[i]} != -* ]]; then
             if [[ ${COMP_WORDS[i-1]} != @($global_args) ]]; then
                 cmd="${COMP_WORDS[i]}"
                 cmd_index=$i
                 break
             fi
         fi
     done
 
     if [[ "$cur" == -* ]]; then
         if [ "$(type -t "_hg_opt_$cmd")" = function ] && "_hg_opt_$cmd"; then
             _hg_fix_wordlist
             return
         fi
 
-        opts=$(_hg_cmd debugcomplete --options "$cmd")
+        opts=$(HGPLAINEXCEPT=alias _hg_cmd debugcomplete --options "$cmd")
 
         COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W '$opts' -- "$cur"))
         _hg_fix_wordlist
         return
     fi
 
     # global options
     case "$prev" in
         -R|--repository)
             _hg_paths
             _hg_repos
             _hg_fix_wordlist
             return
         ;;
         --cwd)
             # Stick with default bash completion
             _hg_fix_wordlist
             return
         ;;
     esac
 
     if [ -z "$cmd" ] || [ $COMP_CWORD -eq $i ]; then
         _hg_commands
         _hg_fix_wordlist
         return
     fi
 
     # try to generate completion candidates for whatever command the user typed
     local help
     if _hg_command_specific; then
         _hg_fix_wordlist
         return
     fi
 
     # canonicalize the command name and try again
     help=$(_hg_cmd help "$cmd")
     if [ $? -ne 0 ]; then
         # Probably either the command doesn't exist or it's ambiguous
         return
     fi
     cmd=${help#hg }
     cmd=${cmd%%[$' \n']*}
     canonical=1
     _hg_command_specific
     _hg_fix_wordlist
 }
 
 _hg_command_specific()
 {
     if [ "$(type -t "_hg_cmd_$cmd")" = function ]; then
         "_hg_cmd_$cmd"
         return 0
     fi
 
     if [ "$cmd" != status ]; then
         case "$prev" in
             -r|--rev)
                 if [[ $canonical = 1 || status != "$cmd"* ]]; then
                     _hg_labels
                     return 0
                 fi
                 return 1
             ;;
             -B|--bookmark)
                 if [[ $canonical = 1 || status != "$cmd"* ]]; then
                     _hg_bookmarks
                     return 0
                 fi
                 return 1
             ;;
             -b|--branch)
                 if [[ $canonical = 1 || status != "$cmd"* ]]; then
                     _hg_branches
                     return 0
                 fi
                 return 1
             ;;
         esac
     fi
 
     local aliascmd=$(_hg_cmd showconfig alias.$cmd | awk '{print $1}')
     [ -n "$aliascmd" ] && cmd=$aliascmd
 
     case "$cmd" in
         help)
             _hg_commands
         ;;
         export)
             if _hg_ext_mq_patchlist qapplied && [ "${COMPREPLY[*]}" ]; then
                 return 0
             fi
             _hg_labels
         ;;
         manifest|update|up|checkout|co)
             _hg_labels
         ;;
         pull|push|outgoing|incoming)
             _hg_paths
             _hg_repos
         ;;
         paths)
             _hg_paths
         ;;
         add)
             _hg_status "u"
         ;;
         merge)
             _hg_labels
         ;;
         commit|ci|record)
             _hg_status "mar"
         ;;
         remove|rm)
             _hg_debugpathcomplete -n
         ;;
         forget)
             _hg_debugpathcomplete -fa
         ;;
         diff)
             _hg_status "mar"
         ;;
         revert)
             _hg_debugpathcomplete
         ;;
         clone)
             local count=$(_hg_count_non_option)
             if [ $count = 1 ]; then
                 _hg_paths
             fi
             _hg_repos
         ;;
         debugindex|debugindexdot)
             COMPREPLY=(${COMPREPLY[@]:-} $(compgen -f -X "!*.i" -- "$cur"))
         ;;
         debugdata)
             COMPREPLY=(${COMPREPLY[@]:-} $(compgen -f -X "!*.d" -- "$cur"))
         ;;
         *)
             return 1
         ;;
     esac
 
     return 0
 }
 
 complete -o bashdefault -o default -o nospace -F _hg hg \
     || complete -o default -o nospace -F _hg hg
 
 
 # Completion for commands provided by extensions
 
 # bookmarks
 _hg_cmd_bookmarks()
 {
     _hg_bookmarks
     return
 }
 
 # mq
 _hg_ext_mq_patchlist()
 {
     local patches
     patches=$(_hg_cmd $1)
     if [ $? -eq 0 ] && [ "$patches" ]; then
         COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W '$patches' -- "$cur"))
         return 0
     fi
     return 1
 }
 
 _hg_ext_mq_queues()
 {
     local root=$(_hg_cmd root)
     local n
     for n in $(cd "$root"/.hg && compgen -d -- "$cur"); do
         # I think we're usually not interested in the regular "patches" queue
         # so just filter it.
         if [ "$n" != patches ] && [ -e "$root/.hg/$n/series" ]; then
             COMPREPLY=(${COMPREPLY[@]:-} "$n")
         fi
     done
 }
 
 _hg_cmd_qpop()
 {
     if [[ "$prev" = @(-n|--name) ]]; then
         _hg_ext_mq_queues
         return
     fi
     _hg_ext_mq_patchlist qapplied
 }
 
 _hg_cmd_qpush()
 {
     if [[ "$prev" = @(-n|--name) ]]; then
         _hg_ext_mq_queues
         return
     fi
     _hg_ext_mq_patchlist qunapplied
 }
 
 _hg_cmd_qgoto()
 {
     if [[ "$prev" = @(-n|--name) ]]; then
         _hg_ext_mq_queues
         return
     fi
     _hg_ext_mq_patchlist qseries
 }
 
 _hg_cmd_qdelete()
 {
     local qcmd=qunapplied
     if [[ "$prev" = @(-r|--rev) ]]; then
         qcmd=qapplied
     fi
     _hg_ext_mq_patchlist $qcmd
 }
 
 _hg_cmd_qfinish()
 {
     if [[ "$prev" = @(-a|--applied) ]]; then
         return
     fi
     _hg_ext_mq_patchlist qapplied
 }
 
 _hg_cmd_qsave()
 {
     if [[ "$prev" = @(-n|--name) ]]; then
         _hg_ext_mq_queues
         return
     fi
 }
 
 _hg_cmd_rebase() {
    if [[ "$prev" = @(-s|--source|-d|--dest|-b|--base|-r|--rev) ]]; then
        _hg_labels
        return
    fi
 }
 
 _hg_cmd_strip()
 {
     if [[ "$prev" = @(-B|--bookmark) ]]; then
         _hg_bookmarks
         return
     fi
     _hg_labels
 }
 
 _hg_cmd_qcommit()
 {
     local root=$(_hg_cmd root)
     # this is run in a sub-shell, so we can't use _hg_status
     local files=$(cd "$root/.hg/patches" && _hg_cmd status -nmar)
     COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W '$files' -- "$cur"))
 }
 
 _hg_cmd_qfold()
 {
     _hg_ext_mq_patchlist qunapplied
 }
 
 _hg_cmd_qrename()
 {
     _hg_ext_mq_patchlist qseries
 }
 
 _hg_cmd_qheader()
 {
     _hg_ext_mq_patchlist qseries
 }
 
 _hg_cmd_qclone()
 {
     local count=$(_hg_count_non_option)
     if [ $count = 1 ]; then
         _hg_paths
     fi
     _hg_repos
 }
 
 _hg_ext_mq_guards()
 {
     _hg_cmd qselect --series | sed -e 's/^.//'
 }
 
 _hg_cmd_qselect()
 {
     local guards=$(_hg_ext_mq_guards)
     COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W '$guards' -- "$cur"))
 }
 
 _hg_cmd_qguard()
 {
     local prefix=''
 
     if [[ "$cur" == +* ]]; then
         prefix=+
     elif [[ "$cur" == -* ]]; then
         prefix=-
     fi
     local ncur=${cur#[-+]}
 
     if ! [ "$prefix" ]; then
         _hg_ext_mq_patchlist qseries
         return
     fi
 
     local guards=$(_hg_ext_mq_guards)
     COMPREPLY=(${COMPREPLY[@]:-} $(compgen -P $prefix -W '$guards' -- "$ncur"))
 }
 
 _hg_opt_qguard()
 {
     local i
     for ((i=cmd_index+1; i<=COMP_CWORD; i++)); do
         if [[ ${COMP_WORDS[i]} != -* ]]; then
             if [[ ${COMP_WORDS[i-1]} != @($global_args) ]]; then
                 _hg_cmd_qguard
                 return 0
             fi
         elif [ "${COMP_WORDS[i]}" = -- ]; then
             _hg_cmd_qguard
             return 0
         fi
     done
     return 1
 }
 
 _hg_cmd_qqueue()
 {
     local q
     local queues
     local opts="--list --create --rename --delete --purge"
 
     queues=$( _hg_cmd qqueue --quiet )
 
     COMPREPLY=( $( compgen -W "${opts} ${queues}" "${cur}" ) )
 }
 
 
 # hbisect
 _hg_cmd_bisect()
 {
     local i subcmd
 
     # find the sub-command
     for ((i=cmd_index+1; i<=COMP_CWORD; i++)); do
         if [[ ${COMP_WORDS[i]} != -* ]]; then
             if [[ ${COMP_WORDS[i-1]} != @($global_args) ]]; then
                 subcmd="${COMP_WORDS[i]}"
                 break
             fi
         fi
     done
 
     if [ -z "$subcmd" ] || [ $COMP_CWORD -eq $i ] || [ "$subcmd" = help ]; then
         COMPREPLY=(${COMPREPLY[@]:-}
                    $(compgen -W 'bad good help init next reset' -- "$cur"))
         return
     fi
 
     case "$subcmd" in
         good|bad)
             _hg_labels
             ;;
     esac
 
     return
 }
 
 
 # patchbomb
 _hg_cmd_email()
 {
     case "$prev" in
         -c|--cc|-t|--to|-f|--from|--bcc)
             # we need an e-mail address. let the user provide a function
             # to get them
             if [ "$(type -t _hg_emails)" = function ]; then
                 local arg=to
                 if [[ "$prev" == @(-f|--from) ]]; then
                     arg=from
                 fi
                 local addresses=$(_hg_emails $arg)
                 COMPREPLY=(${COMPREPLY[@]:-}
                            $(compgen -W '$addresses' -- "$cur"))
             fi
             return
             ;;
         -m|--mbox)
             # fallback to standard filename completion
             return
             ;;
         -s|--subject)
             # free form string
             return
             ;;
     esac
 
     _hg_labels
     return
 }
 
 
 # gpg
 _hg_cmd_sign()
 {
     _hg_labels
 }
 
 
 # transplant
 _hg_cmd_transplant()
 {
     case "$prev" in
         -s|--source)
             _hg_paths
             _hg_repos
             return
             ;;
         --filter)
             # standard filename completion
             return
             ;;
     esac
 
     # all other transplant options values and command parameters are revisions
     _hg_labels
     return
 }
 
 # shelve
 _hg_shelves()
 {
     local shelves="$(_hg_cmd shelve -ql)"
     local IFS=$'\n'
     COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W '$shelves' -- "$cur"))
 }
 
 _hg_cmd_shelve()
 {
     if [[ "$prev" = @(-d|--delete|-l|--list|-p|--patch|--stat) ]]; then
         _hg_shelves
     else
         _hg_status "mard"
     fi
 }
 
 _hg_cmd_unshelve()
 {
     _hg_shelves
 }

contrib/bdiff-torture.py

@@ -1,98 +1,100
 # Randomized torture test generation for bdiff
 
 from __future__ import absolute_import, print_function
-import random, sys
+import random
+import sys
+
 from mercurial import (
     bdiff,
     mpatch,
 )
 
 def reducetest(a, b):
     tries = 0
     reductions = 0
     print("reducing...")
     while tries < 1000:
         a2 = "\n".join(l for l in a.splitlines()
                        if random.randint(0, 100) > 0) + "\n"
         b2 = "\n".join(l for l in b.splitlines()
                        if random.randint(0, 100) > 0) + "\n"
         if a2 == a and b2 == b:
             continue
         if a2 == b2:
             continue
         tries += 1
 
         try:
             test1(a, b)
         except Exception as inst:
             reductions += 1
             tries = 0
             a = a2
             b = b2
 
     print("reduced:", reductions, len(a) + len(b),
           repr(a), repr(b))
     try:
         test1(a, b)
     except Exception as inst:
         print("failed:", inst)
 
     sys.exit(0)
 
 def test1(a, b):
     d = bdiff.bdiff(a, b)
     if not d:
         raise ValueError("empty")
     c = mpatch.patches(a, [d])
     if c != b:
         raise ValueError("bad")
 
 def testwrap(a, b):
     try:
         test1(a, b)
         return
     except Exception as inst:
         pass
     print("exception:", inst)
     reducetest(a, b)
 
 def test(a, b):
     testwrap(a, b)
     testwrap(b, a)
 
 def rndtest(size, noise):
     a = []
     src = "                aaaaaaaabbbbccd"
     for x in xrange(size):
         a.append(src[random.randint(0, len(src) - 1)])
 
     while True:
         b = [c for c in a if random.randint(0, 99) > noise]
         b2 = []
         for c in b:
             b2.append(c)
             while random.randint(0, 99) < noise:
                 b2.append(src[random.randint(0, len(src) - 1)])
         if b2 != a:
             break
 
     a = "\n".join(a) + "\n"
     b = "\n".join(b2) + "\n"
 
     test(a, b)
 
 maxvol = 10000
 startsize = 2
 while True:
     size = startsize
     count = 0
     while size < maxvol:
         print(size)
         volume = 0
         while volume < maxvol:
             rndtest(size, 2)
             volume += size
             count += 2
         size *= 2
     maxvol *= 4
     startsize *= 4

contrib/check-code.py

@@ -1,609 +1,657
 #!/usr/bin/env python
 #
 # check-code - a style and portability checker for Mercurial
 #
 # Copyright 2010 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """style and portability checker for Mercurial
 
 when a rule triggers wrong, do one of the following (prefer one from top):
  * do the work-around the rule suggests
  * doublecheck that it is a false match
  * improve the rule pattern
  * add an ignore pattern to the rule (3rd arg) which matches your good line
    (you can append a short comment and match this, like: #re-raises)
  * change the pattern to a warning and list the exception in test-check-code-hg
  * ONLY use no--check-code for skipping entire files from external sources
 """
 
 from __future__ import absolute_import, print_function
 import glob
 import keyword
 import optparse
 import os
 import re
 import sys
+if sys.version_info[0] < 3:
+    opentext = open
+else:
+    def opentext(f):
+        return open(f, encoding='ascii')
+try:
+    xrange
+except NameError:
+    xrange = range
 try:
     import re2
 except ImportError:
     re2 = None
 
 def compilere(pat, multiline=False):
     if multiline:
         pat = '(?m)' + pat
     if re2:
         try:
             return re2.compile(pat)
         except re2.error:
             pass
     return re.compile(pat)
 
+# check "rules depending on implementation of repquote()" in each
+# patterns (especially pypats), before changing around repquote()
+_repquotefixedmap = {' ': ' ', '\n': '\n', '.': 'p', ':': 'q',
+                     '%': '%', '\\': 'b', '*': 'A', '+': 'P', '-': 'M'}
+def _repquoteencodechr(i):
+    if i > 255:
+        return 'u'
+    c = chr(i)
+    if c in _repquotefixedmap:
+        return _repquotefixedmap[c]
+    if c.isalpha():
+        return 'x'
+    if c.isdigit():
+        return 'n'
+    return 'o'
+_repquotett = ''.join(_repquoteencodechr(i) for i in xrange(256))
+
 def repquote(m):
-    fromc = '.:'
-    tochr = 'pq'
-    def encodechr(i):
-        if i > 255:
-            return 'u'
-        c = chr(i)
-        if c in ' \n':
-            return c
-        if c.isalpha():
-            return 'x'
-        if c.isdigit():
-            return 'n'
-        try:
-            return tochr[fromc.find(c)]
-        except (ValueError, IndexError):
-            return 'o'
     t = m.group('text')
-    tt = ''.join(encodechr(i) for i in xrange(256))
-    t = t.translate(tt)
+    t = t.translate(_repquotett)
     return m.group('quote') + t + m.group('quote')
 
 def reppython(m):
     comment = m.group('comment')
     if comment:
         l = len(comment.rstrip())
         return "#" * l + comment[l:]
     return repquote(m)
 
 def repcomment(m):
     return m.group(1) + "#" * len(m.group(2))
 
 def repccomment(m):
     t = re.sub(r"((?<=\n) )|\S", "x", m.group(2))
     return m.group(1) + t + "*/"
 
 def repcallspaces(m):
     t = re.sub(r"\n\s+", "\n", m.group(2))
     return m.group(1) + t
 
 def repinclude(m):
     return m.group(1) + "<foo>"
 
 def rephere(m):
     t = re.sub(r"\S", "x", m.group(2))
     return m.group(1) + t
 
 
 testpats = [
   [
     (r'pushd|popd', "don't use 'pushd' or 'popd', use 'cd'"),
     (r'\W\$?\(\([^\)\n]*\)\)', "don't use (()) or $(()), use 'expr'"),
     (r'grep.*-q', "don't use 'grep -q', redirect to /dev/null"),
     (r'(?<!hg )grep.* -a', "don't use 'grep -a', use in-line python"),
     (r'sed.*-i', "don't use 'sed -i', use a temporary file"),
     (r'\becho\b.*\\n', "don't use 'echo \\n', use printf"),
     (r'echo -n', "don't use 'echo -n', use printf"),
     (r'(^|\|\s*)\bwc\b[^|]*$\n(?!.*\(re\))', "filter wc output"),
     (r'head -c', "don't use 'head -c', use 'dd'"),
     (r'tail -n', "don't use the '-n' option to tail, just use '-<num>'"),
     (r'sha1sum', "don't use sha1sum, use $TESTDIR/md5sum.py"),
     (r'ls.*-\w*R', "don't use 'ls -R', use 'find'"),
-    (r'printf.*[^\\]\\([1-9]|0\d)', "don't use 'printf \NNN', use Python"),
+    (r'printf.*[^\\]\\([1-9]|0\d)', r"don't use 'printf \NNN', use Python"),
     (r'printf.*[^\\]\\x', "don't use printf \\x, use Python"),
     (r'\$\(.*\)', "don't use $(expr), use `expr`"),
     (r'rm -rf \*', "don't use naked rm -rf, target a directory"),
     (r'(^|\|\s*)grep (-\w\s+)*[^|]*[(|]\w',
      "use egrep for extended grep syntax"),
     (r'/bin/', "don't use explicit paths for tools"),
     (r'[^\n]\Z', "no trailing newline"),
     (r'export .*=', "don't export and assign at once"),
     (r'^source\b', "don't use 'source', use '.'"),
     (r'touch -d', "don't use 'touch -d', use 'touch -t' instead"),
-    (r'ls +[^|\n-]+ +-', "options to 'ls' must come before filenames"),
+    (r'\bls +[^|\n-]+ +-', "options to 'ls' must come before filenames"),
     (r'[^>\n]>\s*\$HGRCPATH', "don't overwrite $HGRCPATH, append to it"),
     (r'^stop\(\)', "don't use 'stop' as a shell function name"),
     (r'(\[|\btest\b).*-e ', "don't use 'test -e', use 'test -f'"),
     (r'\[\[\s+[^\]]*\]\]', "don't use '[[ ]]', use '[ ]'"),
     (r'^alias\b.*=', "don't use alias, use a function"),
     (r'if\s*!', "don't use '!' to negate exit status"),
     (r'/dev/u?random', "don't use entropy, use /dev/zero"),
     (r'do\s*true;\s*done', "don't use true as loop body, use sleep 0"),
     (r'^( *)\t', "don't use tabs to indent"),
     (r'sed (-e )?\'(\d+|/[^/]*/)i(?!\\\n)',
      "put a backslash-escaped newline after sed 'i' command"),
     (r'^diff *-\w*[uU].*$\n(^  \$ |^$)', "prefix diff -u/-U with cmp"),
     (r'^\s+(if)? diff *-\w*[uU]', "prefix diff -u/-U with cmp"),
     (r'seq ', "don't use 'seq', use $TESTDIR/seq.py"),
     (r'\butil\.Abort\b', "directly use error.Abort"),
     (r'\|&', "don't use |&, use 2>&1"),
     (r'\w =  +\w', "only one space after = allowed"),
     (r'\bsed\b.*[^\\]\\n', "don't use 'sed ... \\n', use a \\ and a newline"),
+    (r'env.*-u', "don't use 'env -u VAR', use 'unset VAR'")
   ],
   # warnings
   [
     (r'^function', "don't use 'function', use old style"),
     (r'^diff.*-\w*N', "don't use 'diff -N'"),
     (r'\$PWD|\${PWD}', "don't use $PWD, use `pwd`"),
     (r'^([^"\'\n]|("[^"\n]*")|(\'[^\'\n]*\'))*\^', "^ must be quoted"),
     (r'kill (`|\$\()', "don't use kill, use killdaemons.py")
   ]
 ]
 
 testfilters = [
     (r"( *)(#([^\n]*\S)?)", repcomment),
     (r"<<(\S+)((.|\n)*?\n\1)", rephere),
 ]
 
 winglobmsg = "use (glob) to match Windows paths too"
 uprefix = r"^  \$ "
 utestpats = [
   [
     (r'^(\S.*||  [$>] \S.*)[ \t]\n', "trailing whitespace on non-output"),
     (uprefix + r'.*\|\s*sed[^|>\n]*\n',
      "use regex test output patterns instead of sed"),
     (uprefix + r'(true|exit 0)', "explicit zero exit unnecessary"),
     (uprefix + r'.*(?<!\[)\$\?', "explicit exit code checks unnecessary"),
     (uprefix + r'.*\|\| echo.*(fail|error)',
      "explicit exit code checks unnecessary"),
     (uprefix + r'set -e', "don't use set -e"),
     (uprefix + r'(\s|fi\b|done\b)', "use > for continued lines"),
     (uprefix + r'.*:\.\S*/', "x:.y in a path does not work on msys, rewrite "
      "as x://.y, or see `hg log -k msys` for alternatives", r'-\S+:\.|' #-Rxxx
      '# no-msys'), # in test-pull.t which is skipped on windows
     (r'^  saved backup bundle to \$TESTTMP.*\.hg$', winglobmsg),
     (r'^  changeset .* references (corrupted|missing) \$TESTTMP/.*[^)]$',
      winglobmsg),
     (r'^  pulling from \$TESTTMP/.*[^)]$', winglobmsg,
      '\$TESTTMP/unix-repo$'), # in test-issue1802.t which skipped on windows
     (r'^  reverting (?!subrepo ).*/.*[^)]$', winglobmsg),
     (r'^  cloning subrepo \S+/.*[^)]$', winglobmsg),
     (r'^  pushing to \$TESTTMP/.*[^)]$', winglobmsg),
     (r'^  pushing subrepo \S+/\S+ to.*[^)]$', winglobmsg),
     (r'^  moving \S+/.*[^)]$', winglobmsg),
     (r'^  no changes made to subrepo since.*/.*[^)]$', winglobmsg),
     (r'^  .*: largefile \S+ not available from file:.*/.*[^)]$', winglobmsg),
     (r'^  .*file://\$TESTTMP',
      'write "file:/*/$TESTTMP" + (glob) to match on windows too'),
+    (r'^  [^$>].*27\.0\.0\.1.*[^)]$',
+     'use (glob) to match localhost IP on hosts without 127.0.0.1 too'),
     (r'^  (cat|find): .*: No such file or directory',
      'use test -f to test for file existence'),
     (r'^  diff -[^ -]*p',
      "don't use (external) diff with -p for portability"),
     (r'^  [-+][-+][-+] .* [-+]0000 \(glob\)',
      "glob timezone field in diff output for portability"),
     (r'^  @@ -[0-9]+ [+][0-9]+,[0-9]+ @@',
      "use '@@ -N* +N,n @@ (glob)' style chunk header for portability"),
     (r'^  @@ -[0-9]+,[0-9]+ [+][0-9]+ @@',
      "use '@@ -N,n +N* @@ (glob)' style chunk header for portability"),
     (r'^  @@ -[0-9]+ [+][0-9]+ @@',
      "use '@@ -N* +N* @@ (glob)' style chunk header for portability"),
     (uprefix + r'hg( +-[^ ]+( +[^ ]+)?)* +extdiff'
      r'( +(-[^ po-]+|--(?!program|option)[^ ]+|[^-][^ ]*))*$',
      "use $RUNTESTDIR/pdiff via extdiff (or -o/-p for false-positives)"),
   ],
   # warnings
   [
-    (r'^  [^*?/\n]* \(glob\)$',
-     "glob match with no glob character (?*/)"),
+    (r'^  (?!.*127\.0\.0\.1)[^*?/\n]* \(glob\)$',
+     "glob match with no glob string (?, *, /, and 127.0.0.1)"),
   ]
 ]
 
 for i in [0, 1]:
     for tp in testpats[i]:
         p = tp[0]
         m = tp[1]
         if p.startswith(r'^'):
             p = r"^  [$>] (%s)" % p[1:]
         else:
             p = r"^  [$>] .*(%s)" % p
         utestpats[i].append((p, m) + tp[2:])
 
 utestfilters = [
     (r"<<(\S+)((.|\n)*?\n  > \1)", rephere),
-    (r"( *)(#([^\n]*\S)?)", repcomment),
+    (r"( +)(#([^\n]*\S)?)", repcomment),
 ]
 
 pypats = [
   [
     (r'^\s*def\s*\w+\s*\(.*,\s*\(',
      "tuple parameter unpacking not available in Python 3+"),
     (r'lambda\s*\(.*,.*\)',
      "tuple parameter unpacking not available in Python 3+"),
     (r'(?<!def)\s+(cmp)\(', "cmp is not available in Python 3+"),
     (r'\breduce\s*\(.*', "reduce is not available in Python 3+"),
     (r'dict\(.*=', 'dict() is different in Py2 and 3 and is slower than {}',
      'dict-from-generator'),
     (r'\.has_key\b', "dict.has_key is not available in Python 3+"),
     (r'\s<>\s', '<> operator is not available in Python 3+, use !='),
     (r'^\s*\t', "don't use tabs"),
     (r'\S;\s*\n', "semicolon"),
     (r'[^_]_\([ \t\n]*(?:"[^"]+"[ \t\n+]*)+%', "don't use % inside _()"),
     (r"[^_]_\([ \t\n]*(?:'[^']+'[ \t\n+]*)+%", "don't use % inside _()"),
     (r'(\w|\)),\w', "missing whitespace after ,"),
     (r'(\w|\))[+/*\-<>]\w', "missing whitespace in expression"),
     (r'^\s+(\w|\.)+=\w[^,()\n]*$', "missing whitespace in assignment"),
     (r'\w\s=\s\s+\w', "gratuitous whitespace after ="),
     (r'.{81}', "line too long"),
-    (r' x+[xo][\'"]\n\s+[\'"]x', 'string join across lines with no space'),
     (r'[^\n]\Z', "no trailing newline"),
     (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"),
 #    (r'^\s+[^_ \n][^_. \n]+_[^_\n]+\s*=',
 #     "don't use underbars in identifiers"),
     (r'^\s+(self\.)?[A-za-z][a-z0-9]+[A-Z]\w* = ',
      "don't use camelcase in identifiers"),
     (r'^\s*(if|while|def|class|except|try)\s[^[\n]*:\s*[^\\n]#\s]+',
      "linebreak after :"),
     (r'class\s[^( \n]+:', "old-style class, use class foo(object)",
      r'#.*old-style'),
     (r'class\s[^( \n]+\(\):',
      "class foo() creates old style object, use class foo(object)",
      r'#.*old-style'),
     (r'\b(%s)\(' % '|'.join(k for k in keyword.kwlist
                             if k not in ('print', 'exec')),
      "Python keyword is not a function"),
     (r',]', "unneeded trailing ',' in list"),
 #    (r'class\s[A-Z][^\(]*\((?!Exception)',
 #     "don't capitalize non-exception classes"),
 #    (r'in range\(', "use xrange"),
 #    (r'^\s*print\s+', "avoid using print in core and extensions"),
     (r'[\x80-\xff]', "non-ASCII character literal"),
     (r'("\')\.format\(', "str.format() has no bytes counterpart, use %"),
     (r'^\s*(%s)\s\s' % '|'.join(keyword.kwlist),
      "gratuitous whitespace after Python keyword"),
     (r'([\(\[][ \t]\S)|(\S[ \t][\)\]])', "gratuitous whitespace in () or []"),
 #    (r'\s\s=', "gratuitous whitespace before ="),
     (r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S',
      "missing whitespace around operator"),
     (r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\s',
      "missing whitespace around operator"),
     (r'\s(\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S',
      "missing whitespace around operator"),
     (r'[^^+=*/!<>&| %-](\s=|=\s)[^= ]',
      "wrong whitespace around ="),
     (r'\([^()]*( =[^=]|[^<>!=]= )',
      "no whitespace around = for named parameters"),
     (r'raise Exception', "don't raise generic exceptions"),
     (r'raise [^,(]+, (\([^\)]+\)|[^,\(\)]+)$',
      "don't use old-style two-argument raise, use Exception(message)"),
     (r' is\s+(not\s+)?["\'0-9-]', "object comparison with literal"),
     (r' [=!]=\s+(True|False|None)',
      "comparison with singleton, use 'is' or 'is not' instead"),
     (r'^\s*(while|if) [01]:',
      "use True/False for constant Boolean expression"),
     (r'(?:(?<!def)\s+|\()hasattr',
      'hasattr(foo, bar) is broken, use util.safehasattr(foo, bar) instead'),
     (r'opener\([^)]*\).read\(',
      "use opener.read() instead"),
     (r'opener\([^)]*\).write\(',
      "use opener.write() instead"),
     (r'[\s\(](open|file)\([^)]*\)\.read\(',
      "use util.readfile() instead"),
     (r'[\s\(](open|file)\([^)]*\)\.write\(',
      "use util.writefile() instead"),
     (r'^[\s\(]*(open(er)?|file)\([^)]*\)',
      "always assign an opened file to a variable, and close it afterwards"),
     (r'[\s\(](open|file)\([^)]*\)\.',
      "always assign an opened file to a variable, and close it afterwards"),
     (r'(?i)descend[e]nt', "the proper spelling is descendAnt"),
     (r'\.debug\(\_', "don't mark debug messages for translation"),
     (r'\.strip\(\)\.split\(\)', "no need to strip before splitting"),
     (r'^\s*except\s*:', "naked except clause", r'#.*re-raises'),
     (r'^\s*except\s([^\(,]+|\([^\)]+\))\s*,',
      'legacy exception syntax; use "as" instead of ","'),
     (r':\n(    )*( ){1,3}[^ ]', "must indent 4 spaces"),
-    (r'ui\.(status|progress|write|note|warn)\([\'\"]x',
-     "missing _() in ui message (use () to hide false-positives)"),
     (r'release\(.*wlock, .*lock\)', "wrong lock release order"),
     (r'\b__bool__\b', "__bool__ should be __nonzero__ in Python 2"),
     (r'os\.path\.join\(.*, *(""|\'\')\)',
      "use pathutil.normasprefix(path) instead of os.path.join(path, '')"),
     (r'\s0[0-7]+\b', 'legacy octal syntax; use "0o" prefix instead of "0"'),
     # XXX only catch mutable arguments on the first line of the definition
     (r'def.*[( ]\w+=\{\}', "don't use mutable default arguments"),
     (r'\butil\.Abort\b', "directly use error.Abort"),
     (r'^import Queue', "don't use Queue, use util.queue + util.empty"),
     (r'^import cStringIO', "don't use cStringIO.StringIO, use util.stringio"),
     (r'^import urllib', "don't use urllib, use util.urlreq/util.urlerr"),
+    (r'^import SocketServer', "don't use SockerServer, use util.socketserver"),
+    (r'^import urlparse', "don't use urlparse, use util.urlparse"),
+    (r'^import xmlrpclib', "don't use xmlrpclib, use util.xmlrpclib"),
+    (r'^import cPickle', "don't use cPickle, use util.pickle"),
+    (r'^import pickle', "don't use pickle, use util.pickle"),
+    (r'^import httplib', "don't use httplib, use util.httplib"),
+    (r'^import BaseHTTPServer', "use util.httpserver instead"),
+    (r'\.next\(\)', "don't use .next(), use next(...)"),
+
+    # rules depending on implementation of repquote()
+    (r' x+[xpqo%APM][\'"]\n\s+[\'"]x',
+     'string join across lines with no space'),
+    (r'''(?x)ui\.(status|progress|write|note|warn)\(
+         [ \t\n#]*
+         (?# any strings/comments might precede a string, which
+           # contains translatable message)
+         ((['"]|\'\'\'|""")[ \npq%bAPMxno]*(['"]|\'\'\'|""")[ \t\n#]+)*
+         (?# sequence consisting of below might precede translatable message
+           # - formatting string: "% 10s", "%05d", "% -3.2f", "%*s", "%%" ...
+           # - escaped character: "\\", "\n", "\0" ...
+           # - character other than '%', 'b' as '\', and 'x' as alphabet)
+         (['"]|\'\'\'|""")
+         ((%([ n]?[PM]?([np]+|A))?x)|%%|b[bnx]|[ \nnpqAPMo])*x
+         (?# this regexp can't use [^...] style,
+           # because _preparepats forcibly adds "\n" into [^...],
+           # even though this regexp wants match it against "\n")''',
+     "missing _() in ui message (use () to hide false-positives)"),
   ],
   # warnings
   [
+    # rules depending on implementation of repquote()
     (r'(^| )pp +xxxxqq[ \n][^\n]', "add two newlines after '.. note::'"),
   ]
 ]
 
 pyfilters = [
     (r"""(?msx)(?P<comment>\#.*?$)|
          ((?P<quote>('''|\"\"\"|(?<!')'(?!')|(?<!")"(?!")))
           (?P<text>(([^\\]|\\.)*?))
           (?P=quote))""", reppython),
 ]
 
 txtfilters = []
 
 txtpats = [
   [
     ('\s$', 'trailing whitespace'),
     ('.. note::[ \n][^\n]', 'add two newlines after note::')
   ],
   []
 ]
 
 cpats = [
   [
     (r'//', "don't use //-style comments"),
     (r'^  ', "don't use spaces to indent"),
     (r'\S\t', "don't use tabs except for indent"),
     (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"),
     (r'.{81}', "line too long"),
     (r'(while|if|do|for)\(', "use space after while/if/do/for"),
     (r'return\(', "return is not a function"),
     (r' ;', "no space before ;"),
     (r'[^;] \)', "no space before )"),
     (r'[)][{]', "space between ) and {"),
     (r'\w+\* \w+', "use int *foo, not int* foo"),
     (r'\W\([^\)]+\) \w+', "use (int)foo, not (int) foo"),
     (r'\w+ (\+\+|--)', "use foo++, not foo ++"),
     (r'\w,\w', "missing whitespace after ,"),
     (r'^[^#]\w[+/*]\w', "missing whitespace in expression"),
     (r'\w\s=\s\s+\w', "gratuitous whitespace after ="),
     (r'^#\s+\w', "use #foo, not # foo"),
     (r'[^\n]\Z', "no trailing newline"),
     (r'^\s*#import\b', "use only #include in standard C code"),
     (r'strcpy\(', "don't use strcpy, use strlcpy or memcpy"),
     (r'strcat\(', "don't use strcat"),
+
+    # rules depending on implementation of repquote()
   ],
   # warnings
-  []
+  [
+    # rules depending on implementation of repquote()
+  ]
 ]
 
 cfilters = [
     (r'(/\*)(((\*(?!/))|[^*])*)\*/', repccomment),
     (r'''(?P<quote>(?<!")")(?P<text>([^"]|\\")+)"(?!")''', repquote),
     (r'''(#\s*include\s+<)([^>]+)>''', repinclude),
     (r'(\()([^)]+\))', repcallspaces),
 ]
 
 inutilpats = [
   [
     (r'\bui\.', "don't use ui in util"),
   ],
   # warnings
   []
 ]
 
 inrevlogpats = [
   [
     (r'\brepo\.', "don't use repo in revlog"),
   ],
   # warnings
   []
 ]
 
 webtemplatefilters = []
 
 webtemplatepats = [
   [],
   [
     (r'{desc(\|(?!websub|firstline)[^\|]*)+}',
      'follow desc keyword with either firstline or websub'),
   ]
 ]
 
 checks = [
     ('python', r'.*\.(py|cgi)$', r'^#!.*python', pyfilters, pypats),
     ('test script', r'(.*/)?test-[^.~]*$', '', testfilters, testpats),
     ('c', r'.*\.[ch]$', '', cfilters, cpats),
     ('unified test', r'.*\.t$', '', utestfilters, utestpats),
     ('layering violation repo in revlog', r'mercurial/revlog\.py', '',
      pyfilters, inrevlogpats),
     ('layering violation ui in util', r'mercurial/util\.py', '', pyfilters,
      inutilpats),
     ('txt', r'.*\.txt$', '', txtfilters, txtpats),
     ('web template', r'mercurial/templates/.*\.tmpl', '',
      webtemplatefilters, webtemplatepats),
 ]
 
 def _preparepats():
     for c in checks:
         failandwarn = c[-1]
         for pats in failandwarn:
             for i, pseq in enumerate(pats):
                 # fix-up regexes for multi-line searches
                 p = pseq[0]
                 # \s doesn't match \n
                 p = re.sub(r'(?<!\\)\\s', r'[ \\t]', p)
                 # [^...] doesn't match newline
                 p = re.sub(r'(?<!\\)\[\^', r'[^\\n', p)
 
                 pats[i] = (re.compile(p, re.MULTILINE),) + pseq[1:]
         filters = c[3]
         for i, flt in enumerate(filters):
             filters[i] = re.compile(flt[0]), flt[1]
-_preparepats()
 
 class norepeatlogger(object):
     def __init__(self):
         self._lastseen = None
 
     def log(self, fname, lineno, line, msg, blame):
         """print error related a to given line of a given file.
 
         The faulty line will also be printed but only once in the case
         of multiple errors.
 
         :fname: filename
         :lineno: line number
         :line: actual content of the line
         :msg: error message
         """
         msgid = fname, lineno, line
         if msgid != self._lastseen:
             if blame:
                 print("%s:%d (%s):" % (fname, lineno, blame))
             else:
                 print("%s:%d:" % (fname, lineno))
             print(" > %s" % line)
             self._lastseen = msgid
         print(" " + msg)
 
 _defaultlogger = norepeatlogger()
 
 def getblame(f):
     lines = []
     for l in os.popen('hg annotate -un %s' % f):
         start, line = l.split(':', 1)
         user, rev = start.split()
         lines.append((line[1:-1], user, rev))
     return lines
 
 def checkfile(f, logfunc=_defaultlogger.log, maxerr=None, warnings=False,
               blame=False, debug=False, lineno=True):
     """checks style and portability of a given file
 
     :f: filepath
     :logfunc: function used to report error
               logfunc(filename, linenumber, linecontent, errormessage)
     :maxerr: number of error to display before aborting.
              Set to false (default) to report all errors
 
     return True if no error is found, False otherwise.
     """
     blamecache = None
     result = True
 
     try:
-        fp = open(f)
+        with opentext(f) as fp:
+            try:
+                pre = post = fp.read()
+            except UnicodeDecodeError as e:
+                print("%s while reading %s" % (e, f))
+                return result
     except IOError as e:
         print("Skipping %s, %s" % (f, str(e).split(':', 1)[0]))
         return result
-    pre = post = fp.read()
-    fp.close()
 
     for name, match, magic, filters, pats in checks:
         if debug:
             print(name, f)
         fc = 0
         if not (re.match(match, f) or (magic and re.search(magic, pre))):
             if debug:
                 print("Skipping %s for %s it doesn't match %s" % (
                        name, match, f))
             continue
         if "no-" "check-code" in pre:
             # If you're looking at this line, it's because a file has:
             # no- check- code
             # but the reason to output skipping is to make life for
             # tests easier. So, instead of writing it with a normal
             # spelling, we write it with the expected spelling from
             # tests/test-check-code.t
             print("Skipping %s it has no-che?k-code (glob)" % f)
             return "Skip" # skip checking this file
         for p, r in filters:
             post = re.sub(p, r, post)
         nerrs = len(pats[0]) # nerr elements are errors
         if warnings:
             pats = pats[0] + pats[1]
         else:
             pats = pats[0]
         # print post # uncomment to show filtered version
 
         if debug:
             print("Checking %s for %s" % (name, f))
 
         prelines = None
         errors = []
         for i, pat in enumerate(pats):
             if len(pat) == 3:
                 p, msg, ignore = pat
             else:
                 p, msg = pat
                 ignore = None
             if i >= nerrs:
                 msg = "warning: " + msg
 
             pos = 0
             n = 0
             for m in p.finditer(post):
                 if prelines is None:
                     prelines = pre.splitlines()
                     postlines = post.splitlines(True)
 
                 start = m.start()
                 while n < len(postlines):
                     step = len(postlines[n])
                     if pos + step > start:
                         break
                     pos += step
                     n += 1
                 l = prelines[n]
 
                 if ignore and re.search(ignore, l, re.MULTILINE):
                     if debug:
                         print("Skipping %s for %s:%s (ignore pattern)" % (
                             name, f, n))
                     continue
                 bd = ""
                 if blame:
                     bd = 'working directory'
                     if not blamecache:
                         blamecache = getblame(f)
                     if n < len(blamecache):
                         bl, bu, br = blamecache[n]
                         if bl == l:
                             bd = '%s@%s' % (bu, br)
 
                 errors.append((f, lineno and n + 1, l, msg, bd))
                 result = False
 
         errors.sort()
         for e in errors:
             logfunc(*e)
             fc += 1
             if maxerr and fc >= maxerr:
                 print(" (too many errors, giving up)")
                 break
 
     return result
 
-if __name__ == "__main__":
+def main():
     parser = optparse.OptionParser("%prog [options] [files]")
     parser.add_option("-w", "--warnings", action="store_true",
                       help="include warning-level checks")
     parser.add_option("-p", "--per-file", type="int",
                       help="max warnings per file")
     parser.add_option("-b", "--blame", action="store_true",
                       help="use annotate to generate blame info")
     parser.add_option("", "--debug", action="store_true",
                       help="show debug information")
     parser.add_option("", "--nolineno", action="store_false",
                       dest='lineno', help="don't show line numbers")
 
     parser.set_defaults(per_file=15, warnings=False, blame=False, debug=False,
                         lineno=True)
     (options, args) = parser.parse_args()
 
     if len(args) == 0:
         check = glob.glob("*")
     else:
         check = args
 
+    _preparepats()
+
     ret = 0
     for f in check:
         if not checkfile(f, maxerr=options.per_file, warnings=options.warnings,
                          blame=options.blame, debug=options.debug,
                          lineno=options.lineno):
             ret = 1
-    sys.exit(ret)
+    return ret
+
+if __name__ == "__main__":
+    sys.exit(main())

contrib/check-commit

@@ -1,98 +1,102
 #!/usr/bin/env python
 #
 # Copyright 2014 Matt Mackall <[e-mail]>
 #
 # A tool/hook to run basic sanity checks on commits/patches for
 # submission to Mercurial. Install by adding the following to your
 # .hg/hgrc:
 #
 # [hooks]
 # pretxncommit = contrib/check-commit
 #
 # The hook can be temporarily bypassed with:
 #
 # $ BYPASS= hg commit
 #
 # See also: https://mercurial-scm.org/wiki/ContributingChanges
 
-import re, sys, os
+from __future__ import absolute_import, print_function
+
+import os
+import re
+import sys
 
 commitheader = r"^(?:# [^\n]*\n)*"
 afterheader = commitheader + r"(?!#)"
 beforepatch = afterheader + r"(?!\n(?!@@))"
 
 errors = [
     (beforepatch + r".*[(]bc[)]", "(BC) needs to be uppercase"),
     (beforepatch + r".*[(]issue \d\d\d",
      "no space allowed between issue and number"),
     (beforepatch + r".*[(]bug(\d|\s)", "use (issueDDDD) instead of bug"),
     (commitheader + r"# User [^@\n]+\n", "username is not an email address"),
     (commitheader + r"(?!merge with )[^#]\S+[^:] ",
      "summary line doesn't start with 'topic: '"),
     (afterheader + r"[A-Z][a-z]\S+", "don't capitalize summary lines"),
     (afterheader + r"[^\n]*: *[A-Z][a-z]\S+", "don't capitalize summary lines"),
     (afterheader + r"\S*[^A-Za-z0-9-]\S*: ",
      "summary keyword should be most user-relevant one-word command or topic"),
     (afterheader + r".*\.\s*\n", "don't add trailing period on summary line"),
     (afterheader + r".{79,}", "summary line too long (limit is 78)"),
     (r"\n\+\n( |\+)\n", "adds double empty line"),
     (r"\n \n\+\n", "adds double empty line"),
     (r"\n\+[ \t]+def [a-z]+_[a-z]", "adds a function with foo_bar naming"),
 ]
 
 word = re.compile('\S')
 def nonempty(first, second):
     if word.search(first):
         return first
     return second
 
 def checkcommit(commit, node=None):
     exitcode = 0
     printed = node is None
     hits = []
     for exp, msg in errors:
         for m in re.finditer(exp, commit):
             end = m.end()
             trailing = re.search(r'(\\n)+$', exp)
             if trailing:
                 end -= len(trailing.group()) / 2
             hits.append((end, exp, msg))
     if hits:
         hits.sort()
         pos = 0
         last = ''
         for n, l in enumerate(commit.splitlines(True)):
             pos += len(l)
             while len(hits):
                 end, exp, msg = hits[0]
                 if pos < end:
                     break
                 if not printed:
                     printed = True
-                    print "node: %s" % node
-                print "%d: %s" % (n, msg)
-                print " %s" % nonempty(l, last)[:-1]
+                    print("node: %s" % node)
+                print("%d: %s" % (n, msg))
+                print(" %s" % nonempty(l, last)[:-1])
                 if "BYPASS" not in os.environ:
                     exitcode = 1
                 del hits[0]
             last = nonempty(l, last)
 
     return exitcode
 
 def readcommit(node):
     return os.popen("hg export %s" % node).read()
 
 if __name__ == "__main__":
     exitcode = 0
     node = os.environ.get("HG_NODE")
 
     if node:
         commit = readcommit(node)
         exitcode = checkcommit(commit)
     elif sys.argv[1:]:
         for node in sys.argv[1:]:
             exitcode |= checkcommit(readcommit(node), node)
     else:
         commit = sys.stdin.read()
         exitcode = checkcommit(commit)
     sys.exit(exitcode)

contrib/check-py3-compat.py

@@ -1,83 +1,96
 #!/usr/bin/env python
 #
 # check-py3-compat - check Python 3 compatibility of Mercurial files
 #
 # Copyright 2015 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import, print_function
 
 import ast
 import imp
 import os
 import sys
 import traceback
 
 def check_compat_py2(f):
     """Check Python 3 compatibility for a file with Python 2"""
     with open(f, 'rb') as fh:
         content = fh.read()
     root = ast.parse(content)
 
     # Ignore empty files.
     if not root.body:
         return
 
     futures = set()
     haveprint = False
     for node in ast.walk(root):
         if isinstance(node, ast.ImportFrom):
             if node.module == '__future__':
                 futures |= set(n.name for n in node.names)
         elif isinstance(node, ast.Print):
             haveprint = True
 
     if 'absolute_import' not in futures:
         print('%s not using absolute_import' % f)
     if haveprint and 'print_function' not in futures:
         print('%s requires print_function' % f)
 
 def check_compat_py3(f):
     """Check Python 3 compatibility of a file with Python 3."""
     with open(f, 'rb') as fh:
         content = fh.read()
 
     try:
         ast.parse(content)
     except SyntaxError as e:
         print('%s: invalid syntax: %s' % (f, e))
         return
 
     # Try to import the module.
     # For now we only support mercurial.* and hgext.* modules because figuring
     # out module paths for things not in a package can be confusing.
     if f.startswith(('hgext/', 'mercurial/')) and not f.endswith('__init__.py'):
         assert f.endswith('.py')
         name = f.replace('/', '.')[:-3]
         with open(f, 'r') as fh:
             try:
                 imp.load_module(name, fh, '', ('py', 'r', imp.PY_SOURCE))
             except Exception as e:
                 exc_type, exc_value, tb = sys.exc_info()
-                frame = traceback.extract_tb(tb)[-1]
+                # We walk the stack and ignore frames from our custom importer,
+                # import mechanisms, and stdlib modules. This kinda/sorta
+                # emulates CPython behavior in import.c while also attempting
+                # to pin blame on a Mercurial file.
+                for frame in reversed(traceback.extract_tb(tb)):
+                    if frame.name == '_call_with_frames_removed':
+                        continue
+                    if 'importlib' in frame.filename:
+                        continue
+                    if 'mercurial/__init__.py' in frame.filename:
+                        continue
+                    if frame.filename.startswith(sys.prefix):
+                        continue
+                    break
 
                 if frame.filename:
                     filename = os.path.basename(frame.filename)
                     print('%s: error importing: <%s> %s (error at %s:%d)' % (
                           f, type(e).__name__, e, filename, frame.lineno))
                 else:
                     print('%s: error importing module: <%s> %s (line %d)' % (
                           f, type(e).__name__, e, frame.lineno))
 
 if __name__ == '__main__':
     if sys.version_info[0] == 2:
         fn = check_compat_py2
     else:
         fn = check_compat_py3
 
     for f in sys.argv[1:]:
         fn(f)
 
     sys.exit(0)

contrib/chg/README

@@ -1,30 +1,32
 cHg
 ===
 
 A fast client for Mercurial command server running on Unix.
 
 Install:
 
  $ make
  $ make install
 
 Usage:
 
  $ chg help                 # show help of Mercurial
  $ alias hg=chg             # replace hg command
  $ chg --kill-chg-daemon    # terminate background server
 
 Environment variables:
 
 Although cHg tries to update environment variables, some of them cannot be
 changed after spawning the server. The following variables are specially
 handled:
 
  * configuration files are reloaded automatically by default.
  * CHGHG or HG specifies the path to the hg executable spawned as the
    background command server.
 
 The following variables are available for testing:
 
  * CHGDEBUG enables debug messages.
  * CHGSOCKNAME specifies the socket path of the background cmdserver.
+ * CHGTIMEOUT specifies how many seconds chg will wait before giving up
+   connecting to a cmdserver. If it is 0, chg will wait forever. Default: 60

contrib/chg/chg.c

@@ -1,593 +1,669
 /*
  * A fast client for Mercurial command server
  *
  * Copyright (c) 2011 Yuya Nishihara <[e-mail]>
  *
  * This software may be used and distributed according to the terms of the
  * GNU General Public License version 2 or any later version.
  */
 
 #include <assert.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sys/file.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/un.h>
 #include <sys/wait.h>
 #include <time.h>
 #include <unistd.h>
 
 #include "hgclient.h"
 #include "util.h"
 
 #ifndef UNIX_PATH_MAX
 #define UNIX_PATH_MAX (sizeof(((struct sockaddr_un *)NULL)->sun_path))
 #endif
 
 struct cmdserveropts {
 	char sockname[UNIX_PATH_MAX];
 	char redirectsockname[UNIX_PATH_MAX];
 	char lockfile[UNIX_PATH_MAX];
 	size_t argsize;
 	const char **args;
 	int lockfd;
 	int sockdirfd;
 };
 
 static void initcmdserveropts(struct cmdserveropts *opts) {
 	memset(opts, 0, sizeof(struct cmdserveropts));
 	opts->lockfd = -1;
 	opts->sockdirfd = -1;
 }
 
 static void freecmdserveropts(struct cmdserveropts *opts) {
 	free(opts->args);
 	opts->args = NULL;
 	opts->argsize = 0;
 	assert(opts->lockfd == -1 && "should be closed by unlockcmdserver()");
 	if (opts->sockdirfd >= 0) {
 		close(opts->sockdirfd);
 		opts->sockdirfd = -1;
 	}
 }
 
 /*
  * Test if an argument is a sensitive flag that should be passed to the server.
  * Return 0 if not, otherwise the number of arguments starting from the current
  * one that should be passed to the server.
  */
 static size_t testsensitiveflag(const char *arg)
 {
 	static const struct {
 		const char *name;
 		size_t narg;
 	} flags[] = {
 		{"--config", 1},
 		{"--cwd", 1},
 		{"--repo", 1},
 		{"--repository", 1},
 		{"--traceback", 0},
 		{"-R", 1},
 	};
 	size_t i;
 	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); ++i) {
 		size_t len = strlen(flags[i].name);
 		size_t narg = flags[i].narg;
 		if (memcmp(arg, flags[i].name, len) == 0) {
 			if (arg[len] == '\0') {
 				/* --flag (value) */
 				return narg + 1;
 			} else if (arg[len] == '=' && narg > 0) {
 				/* --flag=value */
 				return 1;
 			} else if (flags[i].name[1] != '-') {
 				/* short flag */
 				return 1;
 			}
 		}
 	}
 	return 0;
 }
 
 /*
  * Parse argv[] and put sensitive flags to opts->args
  */
 static void setcmdserverargs(struct cmdserveropts *opts,
 			     int argc, const char *argv[])
 {
 	size_t i, step;
 	opts->argsize = 0;
 	for (i = 0, step = 1; i < (size_t)argc; i += step, step = 1) {
 		if (!argv[i])
 			continue;  /* pass clang-analyse */
 		if (strcmp(argv[i], "--") == 0)
 			break;
 		size_t n = testsensitiveflag(argv[i]);
 		if (n == 0 || i + n > (size_t)argc)
 			continue;
 		opts->args = reallocx(opts->args,
 				      (n + opts->argsize) * sizeof(char *));
 		memcpy(opts->args + opts->argsize, argv + i,
 		       sizeof(char *) * n);
 		opts->argsize += n;
 		step = n;
 	}
 }
 
 static void preparesockdir(const char *sockdir)
 {
 	int r;
 	r = mkdir(sockdir, 0700);
 	if (r < 0 && errno != EEXIST)
 		abortmsgerrno("cannot create sockdir %s", sockdir);
 
 	struct stat st;
 	r = lstat(sockdir, &st);
 	if (r < 0)
 		abortmsgerrno("cannot stat %s", sockdir);
 	if (!S_ISDIR(st.st_mode))
 		abortmsg("cannot create sockdir %s (file exists)", sockdir);
 	if (st.st_uid != geteuid() || st.st_mode & 0077)
 		abortmsg("insecure sockdir %s", sockdir);
 }
 
 static void setcmdserveropts(struct cmdserveropts *opts)
 {
 	int r;
 	char sockdir[UNIX_PATH_MAX];
 	const char *envsockname = getenv("CHGSOCKNAME");
 	if (!envsockname) {
 		/* by default, put socket file in secure directory
 		 * (permission of socket file may be ignored on some Unices) */
 		const char *tmpdir = getenv("TMPDIR");
 		if (!tmpdir)
 			tmpdir = "/tmp";
 		r = snprintf(sockdir, sizeof(sockdir), "%s/chg%d",
 			     tmpdir, geteuid());
 		if (r < 0 || (size_t)r >= sizeof(sockdir))
 			abortmsg("too long TMPDIR (r = %d)", r);
 		preparesockdir(sockdir);
 	}
 
 	const char *basename = (envsockname) ? envsockname : sockdir;
 	const char *sockfmt = (envsockname) ? "%s" : "%s/server";
 	const char *lockfmt = (envsockname) ? "%s.lock" : "%s/lock";
 	r = snprintf(opts->sockname, sizeof(opts->sockname), sockfmt, basename);
 	if (r < 0 || (size_t)r >= sizeof(opts->sockname))
 		abortmsg("too long TMPDIR or CHGSOCKNAME (r = %d)", r);
 	r = snprintf(opts->lockfile, sizeof(opts->lockfile), lockfmt, basename);
 	if (r < 0 || (size_t)r >= sizeof(opts->lockfile))
 		abortmsg("too long TMPDIR or CHGSOCKNAME (r = %d)", r);
 }
 
 /*
  * Acquire a file lock that indicates a client is trying to start and connect
  * to a server, before executing a command. The lock is released upon exit or
  * explicit unlock. Will block if the lock is held by another process.
  */
 static void lockcmdserver(struct cmdserveropts *opts)
 {
 	if (opts->lockfd == -1) {
 		opts->lockfd = open(opts->lockfile,
 				    O_RDWR | O_CREAT | O_NOFOLLOW, 0600);
 		if (opts->lockfd == -1)
 			abortmsgerrno("cannot create lock file %s",
 				      opts->lockfile);
 		fsetcloexec(opts->lockfd);
 	}
 	int r = flock(opts->lockfd, LOCK_EX);
 	if (r == -1)
 		abortmsgerrno("cannot acquire lock");
 }
 
 /*
  * Release the file lock held by calling lockcmdserver. Will do nothing if
  * lockcmdserver is not called.
  */
 static void unlockcmdserver(struct cmdserveropts *opts)
 {
 	if (opts->lockfd == -1)
 		return;
 	flock(opts->lockfd, LOCK_UN);
 	close(opts->lockfd);
 	opts->lockfd = -1;
 }
 
 static const char *gethgcmd(void)
 {
 	static const char *hgcmd = NULL;
 	if (!hgcmd) {
 		hgcmd = getenv("CHGHG");
 		if (!hgcmd || hgcmd[0] == '\0')
 			hgcmd = getenv("HG");
 		if (!hgcmd || hgcmd[0] == '\0')
 #ifdef HGPATH
 			hgcmd = (HGPATH);
 #else
 			hgcmd = "hg";
 #endif
 	}
 	return hgcmd;
 }
 
 static void execcmdserver(const struct cmdserveropts *opts)
 {
 	const char *hgcmd = gethgcmd();
 
 	const char *baseargv[] = {
 		hgcmd,
 		"serve",
 		"--cmdserver", "chgunix",
 		"--address", opts->sockname,
 		"--daemon-postexec", "chdir:/",
 		"--config", "extensions.chgserver=",
 	};
 	size_t baseargvsize = sizeof(baseargv) / sizeof(baseargv[0]);
 	size_t argsize = baseargvsize + opts->argsize + 1;
 
 	const char **argv = mallocx(sizeof(char *) * argsize);
 	memcpy(argv, baseargv, sizeof(baseargv));
 	memcpy(argv + baseargvsize, opts->args, sizeof(char *) * opts->argsize);
 	argv[argsize - 1] = NULL;
 
 	if (putenv("CHGINTERNALMARK=") != 0)
 		abortmsgerrno("failed to putenv");
 	if (execvp(hgcmd, (char **)argv) < 0)
 		abortmsgerrno("failed to exec cmdserver");
 	free(argv);
 }
 
 /* Retry until we can connect to the server. Give up after some time. */
 static hgclient_t *retryconnectcmdserver(struct cmdserveropts *opts, pid_t pid)
 {
 	static const struct timespec sleepreq = {0, 10 * 1000000};
 	int pst = 0;
 
 	debugmsg("try connect to %s repeatedly", opts->sockname);
-	for (unsigned int i = 0; i < 10 * 100; i++) {
+
+	unsigned int timeoutsec = 60;  /* default: 60 seconds */
+	const char *timeoutenv = getenv("CHGTIMEOUT");
+	if (timeoutenv)
+		sscanf(timeoutenv, "%u", &timeoutsec);
+
+	for (unsigned int i = 0; !timeoutsec || i < timeoutsec * 100; i++) {
 		hgclient_t *hgc = hgc_open(opts->sockname);
 		if (hgc)
 			return hgc;
 
 		if (pid > 0) {
 			/* collect zombie if child process fails to start */
 			int r = waitpid(pid, &pst, WNOHANG);
 			if (r != 0)
 				goto cleanup;
 		}
 
 		nanosleep(&sleepreq, NULL);
 	}
 
 	abortmsg("timed out waiting for cmdserver %s", opts->sockname);
 	return NULL;
 
 cleanup:
 	if (WIFEXITED(pst)) {
 		if (WEXITSTATUS(pst) == 0)
 			abortmsg("could not connect to cmdserver "
 				 "(exited with status 0)");
 		debugmsg("cmdserver exited with status %d", WEXITSTATUS(pst));
 		exit(WEXITSTATUS(pst));
 	} else if (WIFSIGNALED(pst)) {
 		abortmsg("cmdserver killed by signal %d", WTERMSIG(pst));
 	} else {
 		abortmsg("error while waiting for cmdserver");
 	}
 	return NULL;
 }
 
 /* Connect to a cmdserver. Will start a new server on demand. */
 static hgclient_t *connectcmdserver(struct cmdserveropts *opts)
 {
 	const char *sockname = opts->redirectsockname[0] ?
 		opts->redirectsockname : opts->sockname;
 	debugmsg("try connect to %s", sockname);
 	hgclient_t *hgc = hgc_open(sockname);
 	if (hgc)
 		return hgc;
 
 	lockcmdserver(opts);
 	hgc = hgc_open(sockname);
 	if (hgc) {
 		unlockcmdserver(opts);
 		debugmsg("cmdserver is started by another process");
 		return hgc;
 	}
 
 	/* prevent us from being connected to an outdated server: we were
 	 * told by a server to redirect to opts->redirectsockname and that
 	 * address does not work. we do not want to connect to the server
 	 * again because it will probably tell us the same thing. */
 	if (sockname == opts->redirectsockname)
 		unlink(opts->sockname);
 
 	debugmsg("start cmdserver at %s", opts->sockname);
 
 	pid_t pid = fork();
 	if (pid < 0)
 		abortmsg("failed to fork cmdserver process");
 	if (pid == 0) {
 		execcmdserver(opts);
 	} else {
 		hgc = retryconnectcmdserver(opts, pid);
 	}
 
 	unlockcmdserver(opts);
 	return hgc;
 }
 
 static void killcmdserver(const struct cmdserveropts *opts)
 {
 	/* resolve config hash */
 	char *resolvedpath = realpath(opts->sockname, NULL);
 	if (resolvedpath) {
 		unlink(resolvedpath);
 		free(resolvedpath);
 	}
 }
 
+static pid_t pagerpid = 0;
 static pid_t peerpid = 0;
 
 static void forwardsignal(int sig)
 {
 	assert(peerpid > 0);
 	if (kill(peerpid, sig) < 0)
 		abortmsgerrno("cannot kill %d", peerpid);
 	debugmsg("forward signal %d", sig);
 }
 
 static void handlestopsignal(int sig)
 {
 	sigset_t unblockset, oldset;
 	struct sigaction sa, oldsa;
 	if (sigemptyset(&unblockset) < 0)
 		goto error;
 	if (sigaddset(&unblockset, sig) < 0)
 		goto error;
 	memset(&sa, 0, sizeof(sa));
 	sa.sa_handler = SIG_DFL;
 	sa.sa_flags = SA_RESTART;
 	if (sigemptyset(&sa.sa_mask) < 0)
 		goto error;
 
 	forwardsignal(sig);
 	if (raise(sig) < 0)  /* resend to self */
 		goto error;
 	if (sigaction(sig, &sa, &oldsa) < 0)
 		goto error;
 	if (sigprocmask(SIG_UNBLOCK, &unblockset, &oldset) < 0)
 		goto error;
 	/* resent signal will be handled before sigprocmask() returns */
 	if (sigprocmask(SIG_SETMASK, &oldset, NULL) < 0)
 		goto error;
 	if (sigaction(sig, &oldsa, NULL) < 0)
 		goto error;
 	return;
 
 error:
 	abortmsgerrno("failed to handle stop signal");
 }
 
+static void handlechildsignal(int sig UNUSED_)
+{
+	if (peerpid == 0 || pagerpid == 0)
+		return;
+	/* if pager exits, notify the server with SIGPIPE immediately.
+	 * otherwise the server won't get SIGPIPE if it does not write
+	 * anything. (issue5278) */
+	if (waitpid(pagerpid, NULL, WNOHANG) == pagerpid)
+		kill(peerpid, SIGPIPE);
+}
+
 static void setupsignalhandler(pid_t pid)
 {
 	if (pid <= 0)
 		return;
 	peerpid = pid;
 
 	struct sigaction sa;
 	memset(&sa, 0, sizeof(sa));
 	sa.sa_handler = forwardsignal;
 	sa.sa_flags = SA_RESTART;
 	if (sigemptyset(&sa.sa_mask) < 0)
 		goto error;
 
 	if (sigaction(SIGHUP, &sa, NULL) < 0)
 		goto error;
 	if (sigaction(SIGINT, &sa, NULL) < 0)
 		goto error;
 
 	/* terminate frontend by double SIGTERM in case of server freeze */
 	sa.sa_flags |= SA_RESETHAND;
 	if (sigaction(SIGTERM, &sa, NULL) < 0)
 		goto error;
 
 	/* notify the worker about window resize events */
 	sa.sa_flags = SA_RESTART;
 	if (sigaction(SIGWINCH, &sa, NULL) < 0)
 		goto error;
 	/* propagate job control requests to worker */
 	sa.sa_handler = forwardsignal;
 	sa.sa_flags = SA_RESTART;
 	if (sigaction(SIGCONT, &sa, NULL) < 0)
 		goto error;
 	sa.sa_handler = handlestopsignal;
 	sa.sa_flags = SA_RESTART;
 	if (sigaction(SIGTSTP, &sa, NULL) < 0)
 		goto error;
+	/* get notified when pager exits */
+	sa.sa_handler = handlechildsignal;
+	sa.sa_flags = SA_RESTART;
+	if (sigaction(SIGCHLD, &sa, NULL) < 0)
+		goto error;
 
 	return;
 
 error:
 	abortmsgerrno("failed to set up signal handlers");
 }
 
-/* This implementation is based on hgext/pager.py (pre 369741ef7253) */
-static void setuppager(hgclient_t *hgc, const char *const args[],
+static void restoresignalhandler()
+{
+	struct sigaction sa;
+	memset(&sa, 0, sizeof(sa));
+	sa.sa_handler = SIG_DFL;
+	sa.sa_flags = SA_RESTART;
+	if (sigemptyset(&sa.sa_mask) < 0)
+		goto error;
+
+	if (sigaction(SIGHUP, &sa, NULL) < 0)
+		goto error;
+	if (sigaction(SIGTERM, &sa, NULL) < 0)
+		goto error;
+	if (sigaction(SIGWINCH, &sa, NULL) < 0)
+		goto error;
+	if (sigaction(SIGCONT, &sa, NULL) < 0)
+		goto error;
+	if (sigaction(SIGTSTP, &sa, NULL) < 0)
+		goto error;
+	if (sigaction(SIGCHLD, &sa, NULL) < 0)
+		goto error;
+
+	/* ignore Ctrl+C while shutting down to make pager exits cleanly */
+	sa.sa_handler = SIG_IGN;
+	if (sigaction(SIGINT, &sa, NULL) < 0)
+		goto error;
+
+	peerpid = 0;
+	return;
+
+error:
+	abortmsgerrno("failed to restore signal handlers");
+}
+
+/* This implementation is based on hgext/pager.py (post 369741ef7253)
+ * Return 0 if pager is not started, or pid of the pager */
+static pid_t setuppager(hgclient_t *hgc, const char *const args[],
 		       size_t argsize)
 {
 	const char *pagercmd = hgc_getpager(hgc, args, argsize);
 	if (!pagercmd)
-		return;
+		return 0;
 
 	int pipefds[2];
 	if (pipe(pipefds) < 0)
-		return;
+		return 0;
 	pid_t pid = fork();
 	if (pid < 0)
 		goto error;
-	if (pid == 0) {
+	if (pid > 0) {
 		close(pipefds[0]);
 		if (dup2(pipefds[1], fileno(stdout)) < 0)
 			goto error;
 		if (isatty(fileno(stderr))) {
 			if (dup2(pipefds[1], fileno(stderr)) < 0)
 				goto error;
 		}
 		close(pipefds[1]);
 		hgc_attachio(hgc);  /* reattach to pager */
-		return;
+		return pid;
 	} else {
 		dup2(pipefds[0], fileno(stdin));
 		close(pipefds[0]);
 		close(pipefds[1]);
 
 		int r = execlp("/bin/sh", "/bin/sh", "-c", pagercmd, NULL);
 		if (r < 0) {
 			abortmsgerrno("cannot start pager '%s'", pagercmd);
 		}
-		return;
+		return 0;
 	}
 
 error:
 	close(pipefds[0]);
 	close(pipefds[1]);
 	abortmsgerrno("failed to prepare pager");
+	return 0;
+}
+
+static void waitpager(pid_t pid)
+{
+	/* close output streams to notify the pager its input ends */
+	fclose(stdout);
+	fclose(stderr);
+	while (1) {
+		pid_t ret = waitpid(pid, NULL, 0);
+		if (ret == -1 && errno == EINTR)
+			continue;
+		break;
+	}
 }
 
 /* Run instructions sent from the server like unlink and set redirect path
  * Return 1 if reconnect is needed, otherwise 0 */
 static int runinstructions(struct cmdserveropts *opts, const char **insts)
 {
 	int needreconnect = 0;
 	if (!insts)
 		return needreconnect;
 
 	assert(insts);
 	opts->redirectsockname[0] = '\0';
 	const char **pinst;
 	for (pinst = insts; *pinst; pinst++) {
 		debugmsg("instruction: %s", *pinst);
 		if (strncmp(*pinst, "unlink ", 7) == 0) {
 			unlink(*pinst + 7);
 		} else if (strncmp(*pinst, "redirect ", 9) == 0) {
 			int r = snprintf(opts->redirectsockname,
 					 sizeof(opts->redirectsockname),
 					 "%s", *pinst + 9);
 			if (r < 0 || r >= (int)sizeof(opts->redirectsockname))
 				abortmsg("redirect path is too long (%d)", r);
 			needreconnect = 1;
 		} else if (strncmp(*pinst, "exit ", 5) == 0) {
 			int n = 0;
 			if (sscanf(*pinst + 5, "%d", &n) != 1)
 				abortmsg("cannot read the exit code");
 			exit(n);
 		} else if (strcmp(*pinst, "reconnect") == 0) {
 			needreconnect = 1;
 		} else {
 			abortmsg("unknown instruction: %s", *pinst);
 		}
 	}
 	return needreconnect;
 }
 
 /*
  * Test whether the command is unsupported or not. This is not designed to
  * cover all cases. But it's fast, does not depend on the server and does
  * not return false positives.
  */
 static int isunsupported(int argc, const char *argv[])
 {
 	enum {
 		SERVE = 1,
 		DAEMON = 2,
 		SERVEDAEMON = SERVE | DAEMON,
 		TIME = 4,
 	};
 	unsigned int state = 0;
 	int i;
 	for (i = 0; i < argc; ++i) {
 		if (strcmp(argv[i], "--") == 0)
 			break;
 		if (i == 0 && strcmp("serve", argv[i]) == 0)
 			state |= SERVE;
 		else if (strcmp("-d", argv[i]) == 0 ||
 			 strcmp("--daemon", argv[i]) == 0)
 			state |= DAEMON;
 		else if (strcmp("--time", argv[i]) == 0)
 			state |= TIME;
 	}
 	return (state & TIME) == TIME ||
 	       (state & SERVEDAEMON) == SERVEDAEMON;
 }
 
 static void execoriginalhg(const char *argv[])
 {
 	debugmsg("execute original hg");
 	if (execvp(gethgcmd(), (char **)argv) < 0)
 		abortmsgerrno("failed to exec original hg");
 }
 
 int main(int argc, const char *argv[], const char *envp[])
 {
 	if (getenv("CHGDEBUG"))
 		enabledebugmsg();
 
 	if (!getenv("HGPLAIN") && isatty(fileno(stderr)))
 		enablecolor();
 
 	if (getenv("CHGINTERNALMARK"))
 		abortmsg("chg started by chg detected.\n"
 			 "Please make sure ${HG:-hg} is not a symlink or "
 			 "wrapper to chg. Alternatively, set $CHGHG to the "
 			 "path of real hg.");
 
 	if (isunsupported(argc - 1, argv + 1))
 		execoriginalhg(argv);
 
 	struct cmdserveropts opts;
 	initcmdserveropts(&opts);
 	setcmdserveropts(&opts);
 	setcmdserverargs(&opts, argc, argv);
 
 	if (argc == 2) {
 		if (strcmp(argv[1], "--kill-chg-daemon") == 0) {
 			killcmdserver(&opts);
 			return 0;
 		}
 	}
 
 	hgclient_t *hgc;
 	size_t retry = 0;
 	while (1) {
 		hgc = connectcmdserver(&opts);
 		if (!hgc)
 			abortmsg("cannot open hg client");
 		hgc_setenv(hgc, envp);
 		const char **insts = hgc_validate(hgc, argv + 1, argc - 1);
 		int needreconnect = runinstructions(&opts, insts);
 		free(insts);
 		if (!needreconnect)
 			break;
 		hgc_close(hgc);
 		if (++retry > 10)
 			abortmsg("too many redirections.\n"
 				 "Please make sure %s is not a wrapper which "
 				 "changes sensitive environment variables "
 				 "before executing hg. If you have to use a "
 				 "wrapper, wrap chg instead of hg.",
 				 gethgcmd());
 	}
 
 	setupsignalhandler(hgc_peerpid(hgc));
-	setuppager(hgc, argv + 1, argc - 1);
+	pagerpid = setuppager(hgc, argv + 1, argc - 1);
 	int exitcode = hgc_runcommand(hgc, argv + 1, argc - 1);
+	restoresignalhandler();
 	hgc_close(hgc);
 	freecmdserveropts(&opts);
+	if (pagerpid)
+		waitpager(pagerpid);
+
 	return exitcode;
 }

contrib/chg/hgclient.c

@@ -1,576 +1,590
 /*
  * A command server client that uses Unix domain socket
  *
  * Copyright (c) 2011 Yuya Nishihara <[e-mail]>
  *
  * This software may be used and distributed according to the terms of the
  * GNU General Public License version 2 or any later version.
  */
 
 #include <arpa/inet.h>  /* for ntohl(), htonl() */
 #include <assert.h>
 #include <ctype.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <signal.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sys/socket.h>
 #include <sys/stat.h>
 #include <sys/un.h>
 #include <unistd.h>
 
 #include "hgclient.h"
 #include "util.h"
 
 enum {
 	CAP_GETENCODING = 0x0001,
 	CAP_RUNCOMMAND = 0x0002,
 	/* cHg extension: */
 	CAP_ATTACHIO = 0x0100,
 	CAP_CHDIR = 0x0200,
 	CAP_GETPAGER = 0x0400,
 	CAP_SETENV = 0x0800,
 	CAP_SETUMASK = 0x1000,
 	CAP_VALIDATE = 0x2000,
 };
 
 typedef struct {
 	const char *name;
 	unsigned int flag;
 } cappair_t;
 
 static const cappair_t captable[] = {
 	{"getencoding", CAP_GETENCODING},
 	{"runcommand", CAP_RUNCOMMAND},
 	{"attachio", CAP_ATTACHIO},
 	{"chdir", CAP_CHDIR},
 	{"getpager", CAP_GETPAGER},
 	{"setenv", CAP_SETENV},
 	{"setumask", CAP_SETUMASK},
 	{"validate", CAP_VALIDATE},
 	{NULL, 0},  /* terminator */
 };
 
 typedef struct {
 	char ch;
 	char *data;
 	size_t maxdatasize;
 	size_t datasize;
 } context_t;
 
 struct hgclient_tag_ {
 	int sockfd;
+	pid_t pgid;
 	pid_t pid;
 	context_t ctx;
 	unsigned int capflags;
 };
 
 static const size_t defaultdatasize = 4096;
 
 static void initcontext(context_t *ctx)
 {
 	ctx->ch = '\0';
 	ctx->data = malloc(defaultdatasize);
 	ctx->maxdatasize = (ctx->data) ? defaultdatasize : 0;
 	ctx->datasize = 0;
 	debugmsg("initialize context buffer with size %zu", ctx->maxdatasize);
 }
 
 static void enlargecontext(context_t *ctx, size_t newsize)
 {
 	if (newsize <= ctx->maxdatasize)
 		return;
 
 	newsize = defaultdatasize
 		* ((newsize + defaultdatasize - 1) / defaultdatasize);
 	ctx->data = reallocx(ctx->data, newsize);
 	ctx->maxdatasize = newsize;
 	debugmsg("enlarge context buffer to %zu", ctx->maxdatasize);
 }
 
 static void freecontext(context_t *ctx)
 {
 	debugmsg("free context buffer");
 	free(ctx->data);
 	ctx->data = NULL;
 	ctx->maxdatasize = 0;
 	ctx->datasize = 0;
 }
 
 /* Read channeled response from cmdserver */
 static void readchannel(hgclient_t *hgc)
 {
 	assert(hgc);
 
 	ssize_t rsize = recv(hgc->sockfd, &hgc->ctx.ch, sizeof(hgc->ctx.ch), 0);
 	if (rsize != sizeof(hgc->ctx.ch)) {
 		/* server would have exception and traceback would be printed */
 		debugmsg("failed to read channel");
 		exit(255);
 	}
 
 	uint32_t datasize_n;
 	rsize = recv(hgc->sockfd, &datasize_n, sizeof(datasize_n), 0);
 	if (rsize != sizeof(datasize_n))
 		abortmsg("failed to read data size");
 
 	/* datasize denotes the maximum size to write if input request */
 	hgc->ctx.datasize = ntohl(datasize_n);
 	enlargecontext(&hgc->ctx, hgc->ctx.datasize);
 
 	if (isupper(hgc->ctx.ch) && hgc->ctx.ch != 'S')
 		return;  /* assumes input request */
 
 	size_t cursize = 0;
+	int emptycount = 0;
 	while (cursize < hgc->ctx.datasize) {
 		rsize = recv(hgc->sockfd, hgc->ctx.data + cursize,
 			     hgc->ctx.datasize - cursize, 0);
-		if (rsize < 0)
+		/* rsize == 0 normally indicates EOF, while it's also a valid
+		 * packet size for unix socket. treat it as EOF and abort if
+		 * we get many empty responses in a row. */
+		emptycount = (rsize == 0 ? emptycount + 1 : 0);
+		if (rsize < 0 || emptycount > 20)
 			abortmsg("failed to read data block");
 		cursize += rsize;
 	}
 }
 
 static void sendall(int sockfd, const void *data, size_t datasize)
 {
 	const char *p = data;
 	const char *const endp = p + datasize;
 	while (p < endp) {
 		ssize_t r = send(sockfd, p, endp - p, 0);
 		if (r < 0)
 			abortmsgerrno("cannot communicate");
 		p += r;
 	}
 }
 
 /* Write lengh-data block to cmdserver */
 static void writeblock(const hgclient_t *hgc)
 {
 	assert(hgc);
 
 	const uint32_t datasize_n = htonl(hgc->ctx.datasize);
 	sendall(hgc->sockfd, &datasize_n, sizeof(datasize_n));
 
 	sendall(hgc->sockfd, hgc->ctx.data, hgc->ctx.datasize);
 }
 
 static void writeblockrequest(const hgclient_t *hgc, const char *chcmd)
 {
 	debugmsg("request %s, block size %zu", chcmd, hgc->ctx.datasize);
 
 	char buf[strlen(chcmd) + 1];
 	memcpy(buf, chcmd, sizeof(buf) - 1);
 	buf[sizeof(buf) - 1] = '\n';
 	sendall(hgc->sockfd, buf, sizeof(buf));
 
 	writeblock(hgc);
 }
 
 /* Build '\0'-separated list of args. argsize < 0 denotes that args are
  * terminated by NULL. */
 static void packcmdargs(context_t *ctx, const char *const args[],
 			ssize_t argsize)
 {
 	ctx->datasize = 0;
 	const char *const *const end = (argsize >= 0) ? args + argsize : NULL;
 	for (const char *const *it = args; it != end && *it; ++it) {
 		const size_t n = strlen(*it) + 1;  /* include '\0' */
 		enlargecontext(ctx, ctx->datasize + n);
 		memcpy(ctx->data + ctx->datasize, *it, n);
 		ctx->datasize += n;
 	}
 
 	if (ctx->datasize > 0)
 		--ctx->datasize;  /* strip last '\0' */
 }
 
 /* Extract '\0'-separated list of args to new buffer, terminated by NULL */
 static const char **unpackcmdargsnul(const context_t *ctx)
 {
 	const char **args = NULL;
 	size_t nargs = 0, maxnargs = 0;
 	const char *s = ctx->data;
 	const char *e = ctx->data + ctx->datasize;
 	for (;;) {
 		if (nargs + 1 >= maxnargs) {  /* including last NULL */
 			maxnargs += 256;
 			args = reallocx(args, maxnargs * sizeof(args[0]));
 		}
 		args[nargs] = s;
 		nargs++;
 		s = memchr(s, '\0', e - s);
 		if (!s)
 			break;
 		s++;
 	}
 	args[nargs] = NULL;
 	return args;
 }
 
 static void handlereadrequest(hgclient_t *hgc)
 {
 	context_t *ctx = &hgc->ctx;
 	size_t r = fread(ctx->data, sizeof(ctx->data[0]), ctx->datasize, stdin);
 	ctx->datasize = r;
 	writeblock(hgc);
 }
 
 /* Read single-line */
 static void handlereadlinerequest(hgclient_t *hgc)
 {
 	context_t *ctx = &hgc->ctx;
 	if (!fgets(ctx->data, ctx->datasize, stdin))
 		ctx->data[0] = '\0';
 	ctx->datasize = strlen(ctx->data);
 	writeblock(hgc);
 }
 
 /* Execute the requested command and write exit code */
 static void handlesystemrequest(hgclient_t *hgc)
 {
 	context_t *ctx = &hgc->ctx;
 	enlargecontext(ctx, ctx->datasize + 1);
 	ctx->data[ctx->datasize] = '\0';  /* terminate last string */
 
 	const char **args = unpackcmdargsnul(ctx);
 	if (!args[0] || !args[1])
 		abortmsg("missing command or cwd in system request");
 	debugmsg("run '%s' at '%s'", args[0], args[1]);
 	int32_t r = runshellcmd(args[0], args + 2, args[1]);
 	free(args);
 
 	uint32_t r_n = htonl(r);
 	memcpy(ctx->data, &r_n, sizeof(r_n));
 	ctx->datasize = sizeof(r_n);
 	writeblock(hgc);
 }
 
 /* Read response of command execution until receiving 'r'-esult */
 static void handleresponse(hgclient_t *hgc)
 {
 	for (;;) {
 		readchannel(hgc);
 		context_t *ctx = &hgc->ctx;
 		debugmsg("response read from channel %c, size %zu",
 			 ctx->ch, ctx->datasize);
 		switch (ctx->ch) {
 		case 'o':
 			fwrite(ctx->data, sizeof(ctx->data[0]), ctx->datasize,
 			       stdout);
 			break;
 		case 'e':
 			fwrite(ctx->data, sizeof(ctx->data[0]), ctx->datasize,
 			       stderr);
 			break;
 		case 'd':
 			/* assumes last char is '\n' */
 			ctx->data[ctx->datasize - 1] = '\0';
 			debugmsg("server: %s", ctx->data);
 			break;
 		case 'r':
 			return;
 		case 'I':
 			handlereadrequest(hgc);
 			break;
 		case 'L':
 			handlereadlinerequest(hgc);
 			break;
 		case 'S':
 			handlesystemrequest(hgc);
 			break;
 		default:
 			if (isupper(ctx->ch))
 				abortmsg("cannot handle response (ch = %c)",
 					 ctx->ch);
 		}
 	}
 }
 
 static unsigned int parsecapabilities(const char *s, const char *e)
 {
 	unsigned int flags = 0;
 	while (s < e) {
 		const char *t = strchr(s, ' ');
 		if (!t || t > e)
 			t = e;
 		const cappair_t *cap;
 		for (cap = captable; cap->flag; ++cap) {
 			size_t n = t - s;
 			if (strncmp(s, cap->name, n) == 0 &&
 			    strlen(cap->name) == n) {
 				flags |= cap->flag;
 				break;
 			}
 		}
 		s = t + 1;
 	}
 	return flags;
 }
 
 static void readhello(hgclient_t *hgc)
 {
 	readchannel(hgc);
 	context_t *ctx = &hgc->ctx;
 	if (ctx->ch != 'o') {
 		char ch = ctx->ch;
 		if (ch == 'e') {
 			/* write early error and will exit */
 			fwrite(ctx->data, sizeof(ctx->data[0]), ctx->datasize,
 			       stderr);
 			handleresponse(hgc);
 		}
 		abortmsg("unexpected channel of hello message (ch = %c)", ch);
 	}
 	enlargecontext(ctx, ctx->datasize + 1);
 	ctx->data[ctx->datasize] = '\0';
 	debugmsg("hello received: %s (size = %zu)", ctx->data, ctx->datasize);
 
 	const char *s = ctx->data;
 	const char *const dataend = ctx->data + ctx->datasize;
 	while (s < dataend) {
 		const char *t = strchr(s, ':');
 		if (!t || t[1] != ' ')
 			break;
 		const char *u = strchr(t + 2, '\n');
 		if (!u)
 			u = dataend;
 		if (strncmp(s, "capabilities:", t - s + 1) == 0) {
 			hgc->capflags = parsecapabilities(t + 2, u);
+		} else if (strncmp(s, "pgid:", t - s + 1) == 0) {
+			hgc->pgid = strtol(t + 2, NULL, 10);
 		} else if (strncmp(s, "pid:", t - s + 1) == 0) {
 			hgc->pid = strtol(t + 2, NULL, 10);
 		}
 		s = u + 1;
 	}
 	debugmsg("capflags=0x%04x, pid=%d", hgc->capflags, hgc->pid);
 }
 
 static void attachio(hgclient_t *hgc)
 {
 	debugmsg("request attachio");
 	static const char chcmd[] = "attachio\n";
 	sendall(hgc->sockfd, chcmd, sizeof(chcmd) - 1);
 	readchannel(hgc);
 	context_t *ctx = &hgc->ctx;
 	if (ctx->ch != 'I')
 		abortmsg("unexpected response for attachio (ch = %c)", ctx->ch);
 
 	static const int fds[3] = {STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO};
 	struct msghdr msgh;
 	memset(&msgh, 0, sizeof(msgh));
 	struct iovec iov = {ctx->data, ctx->datasize};  /* dummy payload */
 	msgh.msg_iov = &iov;
 	msgh.msg_iovlen = 1;
 	char fdbuf[CMSG_SPACE(sizeof(fds))];
 	msgh.msg_control = fdbuf;
 	msgh.msg_controllen = sizeof(fdbuf);
 	struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msgh);
 	cmsg->cmsg_level = SOL_SOCKET;
 	cmsg->cmsg_type = SCM_RIGHTS;
 	cmsg->cmsg_len = CMSG_LEN(sizeof(fds));
 	memcpy(CMSG_DATA(cmsg), fds, sizeof(fds));
 	msgh.msg_controllen = cmsg->cmsg_len;
 	ssize_t r = sendmsg(hgc->sockfd, &msgh, 0);
 	if (r < 0)
 		abortmsgerrno("sendmsg failed");
 
 	handleresponse(hgc);
 	int32_t n;
 	if (ctx->datasize != sizeof(n))
 		abortmsg("unexpected size of attachio result");
 	memcpy(&n, ctx->data, sizeof(n));
 	n = ntohl(n);
 	if (n != sizeof(fds) / sizeof(fds[0]))
 		abortmsg("failed to send fds (n = %d)", n);
 }
 
 static void chdirtocwd(hgclient_t *hgc)
 {
 	if (!getcwd(hgc->ctx.data, hgc->ctx.maxdatasize))
 		abortmsgerrno("failed to getcwd");
 	hgc->ctx.datasize = strlen(hgc->ctx.data);
 	writeblockrequest(hgc, "chdir");
 }
 
 static void forwardumask(hgclient_t *hgc)
 {
 	mode_t mask = umask(0);
 	umask(mask);
 
 	static const char command[] = "setumask\n";
 	sendall(hgc->sockfd, command, sizeof(command) - 1);
 	uint32_t data = htonl(mask);
 	sendall(hgc->sockfd, &data, sizeof(data));
 }
 
 /*!
  * Open connection to per-user cmdserver
  *
  * If no background server running, returns NULL.
  */
 hgclient_t *hgc_open(const char *sockname)
 {
 	int fd = socket(AF_UNIX, SOCK_STREAM, 0);
 	if (fd < 0)
 		abortmsgerrno("cannot create socket");
 
 	/* don't keep fd on fork(), so that it can be closed when the parent
 	 * process get terminated. */
 	fsetcloexec(fd);
 
 	struct sockaddr_un addr;
 	addr.sun_family = AF_UNIX;
 	strncpy(addr.sun_path, sockname, sizeof(addr.sun_path));
 	addr.sun_path[sizeof(addr.sun_path) - 1] = '\0';
 
 	int r = connect(fd, (struct sockaddr *)&addr, sizeof(addr));
 	if (r < 0) {
 		close(fd);
 		if (errno == ENOENT || errno == ECONNREFUSED)
 			return NULL;
 		abortmsgerrno("cannot connect to %s", addr.sun_path);
 	}
 	debugmsg("connected to %s", addr.sun_path);
 
 	hgclient_t *hgc = mallocx(sizeof(hgclient_t));
 	memset(hgc, 0, sizeof(*hgc));
 	hgc->sockfd = fd;
 	initcontext(&hgc->ctx);
 
 	readhello(hgc);
 	if (!(hgc->capflags & CAP_RUNCOMMAND))
 		abortmsg("insufficient capability: runcommand");
 	if (hgc->capflags & CAP_ATTACHIO)
 		attachio(hgc);
 	if (hgc->capflags & CAP_CHDIR)
 		chdirtocwd(hgc);
 	if (hgc->capflags & CAP_SETUMASK)
 		forwardumask(hgc);
 
 	return hgc;
 }
 
 /*!
  * Close connection and free allocated memory
  */
 void hgc_close(hgclient_t *hgc)
 {
 	assert(hgc);
 	freecontext(&hgc->ctx);
 	close(hgc->sockfd);
 	free(hgc);
 }
 
+pid_t hgc_peerpgid(const hgclient_t *hgc)
+{
+	assert(hgc);
+	return hgc->pgid;
+}
+
 pid_t hgc_peerpid(const hgclient_t *hgc)
 {
 	assert(hgc);
 	return hgc->pid;
 }
 
 /*!
  * Send command line arguments to let the server load the repo config and check
  * whether it can process our request directly or not.
  * Make sure hgc_setenv is called before calling this.
  *
  * @return - NULL, the server believes it can handle our request, or does not
  *           support "validate" command.
  *         - a list of strings, the server probably cannot handle our request
  *           and it sent instructions telling us what to do next. See
  *           chgserver.py for possible instruction formats.
  *           the list should be freed by the caller.
  *           the last string is guaranteed to be NULL.
  */
 const char **hgc_validate(hgclient_t *hgc, const char *const args[],
 			  size_t argsize)
 {
 	assert(hgc);
 	if (!(hgc->capflags & CAP_VALIDATE))
 		return NULL;
 
 	packcmdargs(&hgc->ctx, args, argsize);
 	writeblockrequest(hgc, "validate");
 	handleresponse(hgc);
 
 	/* the server returns '\0' if it can handle our request */
 	if (hgc->ctx.datasize <= 1)
 		return NULL;
 
 	/* make sure the buffer is '\0' terminated */
 	enlargecontext(&hgc->ctx, hgc->ctx.datasize + 1);
 	hgc->ctx.data[hgc->ctx.datasize] = '\0';
 	return unpackcmdargsnul(&hgc->ctx);
 }
 
 /*!
  * Execute the specified Mercurial command
  *
  * @return result code
  */
 int hgc_runcommand(hgclient_t *hgc, const char *const args[], size_t argsize)
 {
 	assert(hgc);
 
 	packcmdargs(&hgc->ctx, args, argsize);
 	writeblockrequest(hgc, "runcommand");
 	handleresponse(hgc);
 
 	int32_t exitcode_n;
 	if (hgc->ctx.datasize != sizeof(exitcode_n)) {
 		abortmsg("unexpected size of exitcode");
 	}
 	memcpy(&exitcode_n, hgc->ctx.data, sizeof(exitcode_n));
 	return ntohl(exitcode_n);
 }
 
 /*!
  * (Re-)send client's stdio channels so that the server can access to tty
  */
 void hgc_attachio(hgclient_t *hgc)
 {
 	assert(hgc);
 	if (!(hgc->capflags & CAP_ATTACHIO))
 		return;
 	attachio(hgc);
 }
 
 /*!
  * Get pager command for the given Mercurial command args
  *
  * If no pager enabled, returns NULL. The return value becomes invalid
  * once you run another request to hgc.
  */
 const char *hgc_getpager(hgclient_t *hgc, const char *const args[],
 			 size_t argsize)
 {
 	assert(hgc);
 
 	if (!(hgc->capflags & CAP_GETPAGER))
 		return NULL;
 
 	packcmdargs(&hgc->ctx, args, argsize);
 	writeblockrequest(hgc, "getpager");
 	handleresponse(hgc);
 
 	if (hgc->ctx.datasize < 1 || hgc->ctx.data[0] == '\0')
 		return NULL;
 	enlargecontext(&hgc->ctx, hgc->ctx.datasize + 1);
 	hgc->ctx.data[hgc->ctx.datasize] = '\0';
 	return hgc->ctx.data;
 }
 
 /*!
  * Update server's environment variables
  *
  * @param envp  list of environment variables in "NAME=VALUE" format,
  *              terminated by NULL.
  */
 void hgc_setenv(hgclient_t *hgc, const char *const envp[])
 {
 	assert(hgc && envp);
 	if (!(hgc->capflags & CAP_SETENV))
 		return;
 	packcmdargs(&hgc->ctx, envp, /*argsize*/ -1);
 	writeblockrequest(hgc, "setenv");
 }

contrib/chg/hgclient.h

@@ -1,31 +1,32
 /*
  * A command server client that uses Unix domain socket
  *
  * Copyright (c) 2011 Yuya Nishihara <[e-mail]>
  *
  * This software may be used and distributed according to the terms of the
  * GNU General Public License version 2 or any later version.
  */
 
 #ifndef HGCLIENT_H_
 #define HGCLIENT_H_
 
 #include <sys/types.h>
 
 struct hgclient_tag_;
 typedef struct hgclient_tag_ hgclient_t;
 
 hgclient_t *hgc_open(const char *sockname);
 void hgc_close(hgclient_t *hgc);
 
+pid_t hgc_peerpgid(const hgclient_t *hgc);
 pid_t hgc_peerpid(const hgclient_t *hgc);
 
 const char **hgc_validate(hgclient_t *hgc, const char *const args[],
 			  size_t argsize);
 int hgc_runcommand(hgclient_t *hgc, const char *const args[], size_t argsize);
 void hgc_attachio(hgclient_t *hgc);
 const char *hgc_getpager(hgclient_t *hgc, const char *const args[],
 			 size_t argsize);
 void hgc_setenv(hgclient_t *hgc, const char *const envp[]);
 
 #endif  /* HGCLIENT_H_ */

contrib/chg/util.h

@@ -1,33 +1,35
 /*
  * Utility functions
  *
  * Copyright (c) 2011 Yuya Nishihara <[e-mail]>
  *
  * This software may be used and distributed according to the terms of the
  * GNU General Public License version 2 or any later version.
  */
 
 #ifndef UTIL_H_
 #define UTIL_H_
 
 #ifdef __GNUC__
 #define PRINTF_FORMAT_ __attribute__((format(printf, 1, 2)))
+#define UNUSED_ __attribute__((unused))
 #else
 #define PRINTF_FORMAT_
+#define UNUSED_
 #endif
 
 void abortmsg(const char *fmt, ...) PRINTF_FORMAT_;
 void abortmsgerrno(const char *fmt, ...) PRINTF_FORMAT_;
 
 void enablecolor(void);
 void enabledebugmsg(void);
 void debugmsg(const char *fmt, ...) PRINTF_FORMAT_;
 
 void fchdirx(int dirfd);
 void fsetcloexec(int fd);
 void *mallocx(size_t size);
 void *reallocx(void *ptr, size_t size);
 
 int runshellcmd(const char *cmd, const char *envp[], const char *cwd);
 
 #endif  /* UTIL_H_ */

contrib/debugshell.py

@@ -1,59 +1,59
 # debugshell extension
 """a python shell with repo, changelog & manifest objects"""
 
 from __future__ import absolute_import
 import code
 import mercurial
 import sys
 from mercurial import (
     cmdutil,
     demandimport,
 )
 
 cmdtable = {}
 command = cmdutil.command(cmdtable)
 
 def pdb(ui, repo, msg, **opts):
     objects = {
         'mercurial': mercurial,
         'repo': repo,
         'cl': repo.changelog,
         'mf': repo.manifest,
     }
 
     code.interact(msg, local=objects)
 
 def ipdb(ui, repo, msg, **opts):
     import IPython
 
     cl = repo.changelog
     mf = repo.manifest
     cl, mf # use variables to appease pyflakes
 
     IPython.embed()
 
 @command('debugshell|dbsh', [])
 def debugshell(ui, repo, **opts):
     bannermsg = "loaded repo : %s\n" \
                 "using source: %s" % (repo.root,
                                       mercurial.__path__[0])
 
     pdbmap = {
         'pdb'  : 'code',
         'ipdb' : 'IPython'
     }
 
     debugger = ui.config("ui", "debugger")
     if not debugger:
         debugger = 'pdb'
 
     # if IPython doesn't exist, fallback to code.interact
     try:
         with demandimport.deactivated():
             __import__(pdbmap[debugger])
     except ImportError:
-        ui.warn("%s debugger specified but %s module was not found\n"
+        ui.warn(("%s debugger specified but %s module was not found\n")
                 % (debugger, pdbmap[debugger]))
         debugger = 'pdb'
 
     getattr(sys.modules[__name__], debugger)(ui, repo, bannermsg, **opts)

contrib/dirstatenonnormalcheck.py

@@ -1,57 +1,57
 # dirstatenonnormalcheck.py - extension to check the consistency of the
 # dirstate's non-normal map
 #
 # For most operations on dirstate, this extensions checks that the nonnormalset
 # contains the right entries.
 # It compares the nonnormal file to a nonnormalset built from the map of all
 # the files in the dirstate to check that they contain the same files.
 
 from __future__ import absolute_import
 
 from mercurial import (
     dirstate,
     extensions,
 )
 
 def nonnormalentries(dmap):
     """Compute nonnormal entries from dirstate's dmap"""
     res = set()
     for f, e in dmap.iteritems():
         if e[0] != 'n' or e[3] == -1:
             res.add(f)
     return res
 
 def checkconsistency(ui, orig, dmap, _nonnormalset, label):
     """Compute nonnormalset from dmap, check that it matches _nonnormalset"""
     nonnormalcomputedmap = nonnormalentries(dmap)
     if _nonnormalset != nonnormalcomputedmap:
-        ui.develwarn("%s call to %s\n" % (label, orig))
-        ui.develwarn("inconsistency in nonnormalset\n")
-        ui.develwarn("[nonnormalset] %s\n" % _nonnormalset)
-        ui.develwarn("[map] %s\n" % nonnormalcomputedmap)
+        ui.develwarn("%s call to %s\n" % (label, orig), config='dirstate')
+        ui.develwarn("inconsistency in nonnormalset\n", config='dirstate')
+        ui.develwarn("[nonnormalset] %s\n" % _nonnormalset, config='dirstate')
+        ui.develwarn("[map] %s\n" % nonnormalcomputedmap, config='dirstate')
 
 def _checkdirstate(orig, self, arg):
     """Check nonnormal set consistency before and after the call to orig"""
     checkconsistency(self._ui, orig, self._map, self._nonnormalset, "before")
     r = orig(self, arg)
     checkconsistency(self._ui, orig, self._map, self._nonnormalset, "after")
     return r
 
 def extsetup(ui):
     """Wrap functions modifying dirstate to check nonnormalset consistency"""
     dirstatecl = dirstate.dirstate
     devel = ui.configbool('devel', 'all-warnings')
     paranoid = ui.configbool('experimental', 'nonnormalparanoidcheck')
     if devel:
         extensions.wrapfunction(dirstatecl, '_writedirstate', _checkdirstate)
         if paranoid:
             # We don't do all these checks when paranoid is disable as it would
             # make the extension run very slowly on large repos
             extensions.wrapfunction(dirstatecl, 'normallookup', _checkdirstate)
             extensions.wrapfunction(dirstatecl, 'otherparent', _checkdirstate)
             extensions.wrapfunction(dirstatecl, 'normal', _checkdirstate)
             extensions.wrapfunction(dirstatecl, 'write', _checkdirstate)
             extensions.wrapfunction(dirstatecl, 'add', _checkdirstate)
             extensions.wrapfunction(dirstatecl, 'remove', _checkdirstate)
             extensions.wrapfunction(dirstatecl, 'merge', _checkdirstate)
             extensions.wrapfunction(dirstatecl, 'drop', _checkdirstate)

contrib/dumprevlog

@@ -1,25 +1,31
 #!/usr/bin/env python
 # Dump revlogs as raw data stream
 # $ find .hg/store/ -name "*.i" | xargs dumprevlog > repo.dump
 
+from __future__ import absolute_import, print_function
+
 import sys
-from mercurial import revlog, node, util
+from mercurial import (
+    node,
+    revlog,
+    util,
+)
 
 for fp in (sys.stdin, sys.stdout, sys.stderr):
     util.setbinary(fp)
 
 for f in sys.argv[1:]:
     binopen = lambda fn: open(fn, 'rb')
     r = revlog.revlog(binopen, f)
-    print "file:", f
+    print("file:", f)
     for i in r:
         n = r.node(i)
         p = r.parents(n)
         d = r.revision(n)
-        print "node:", node.hex(n)
-        print "linkrev:", r.linkrev(i)
-        print "parents:", node.hex(p[0]), node.hex(p[1])
-        print "length:", len(d)
-        print "-start-"
-        print d
-        print "-end-"
+        print("node:", node.hex(n))
+        print("linkrev:", r.linkrev(i))
+        print("parents:", node.hex(p[0]), node.hex(p[1]))
+        print("length:", len(d))
+        print("-start-")
+        print(d)
+        print("-end-")

contrib/import-checker.py

@@ -1,706 +1,722
 #!/usr/bin/env python
 
 from __future__ import absolute_import, print_function
 
 import ast
 import collections
 import os
 import re
 import sys
 
 # Import a minimal set of stdlib modules needed for list_stdlib_modules()
 # to work when run from a virtualenv.  The modules were chosen empirically
 # so that the return value matches the return value without virtualenv.
-import BaseHTTPServer
-import zlib
+if True: # disable lexical sorting checks
+    import BaseHTTPServer
+    import zlib
 
 # Whitelist of modules that symbols can be directly imported from.
 allowsymbolimports = (
     '__future__',
     'mercurial.hgweb.common',
     'mercurial.hgweb.request',
     'mercurial.i18n',
     'mercurial.node',
 )
 
 # Modules that must be aliased because they are commonly confused with
 # common variables and can create aliasing and readability issues.
 requirealias = {
     'ui': 'uimod',
 }
 
 def usingabsolute(root):
     """Whether absolute imports are being used."""
     if sys.version_info[0] >= 3:
         return True
 
     for node in ast.walk(root):
         if isinstance(node, ast.ImportFrom):
             if node.module == '__future__':
                 for n in node.names:
                     if n.name == 'absolute_import':
                         return True
 
     return False
 
 def walklocal(root):
     """Recursively yield all descendant nodes but not in a different scope"""
     todo = collections.deque(ast.iter_child_nodes(root))
     yield root, False
     while todo:
         node = todo.popleft()
         newscope = isinstance(node, ast.FunctionDef)
         if not newscope:
             todo.extend(ast.iter_child_nodes(node))
         yield node, newscope
 
 def dotted_name_of_path(path, trimpure=False):
     """Given a relative path to a source file, return its dotted module name.
 
     >>> dotted_name_of_path('mercurial/error.py')
     'mercurial.error'
     >>> dotted_name_of_path('mercurial/pure/parsers.py', trimpure=True)
     'mercurial.parsers'
     >>> dotted_name_of_path('zlibmodule.so')
     'zlib'
     """
     parts = path.replace(os.sep, '/').split('/')
     parts[-1] = parts[-1].split('.', 1)[0] # remove .py and .so and .ARCH.so
     if parts[-1].endswith('module'):
         parts[-1] = parts[-1][:-6]
     if trimpure:
         return '.'.join(p for p in parts if p != 'pure')
     return '.'.join(parts)
 
 def fromlocalfunc(modulename, localmods):
     """Get a function to examine which locally defined module the
     target source imports via a specified name.
 
     `modulename` is an `dotted_name_of_path()`-ed source file path,
     which may have `.__init__` at the end of it, of the target source.
 
     `localmods` is a dict (or set), of which key is an absolute
     `dotted_name_of_path()`-ed source file path of locally defined (=
     Mercurial specific) modules.
 
     This function assumes that module names not existing in
     `localmods` are from the Python standard library.
 
     This function returns the function, which takes `name` argument,
     and returns `(absname, dottedpath, hassubmod)` tuple if `name`
     matches against locally defined module. Otherwise, it returns
     False.
 
     It is assumed that `name` doesn't have `.__init__`.
 
     `absname` is an absolute module name of specified `name`
     (e.g. "hgext.convert"). This can be used to compose prefix for sub
     modules or so.
 
     `dottedpath` is a `dotted_name_of_path()`-ed source file path
     (e.g. "hgext.convert.__init__") of `name`. This is used to look
     module up in `localmods` again.
 
     `hassubmod` is whether it may have sub modules under it (for
     convenient, even though this is also equivalent to "absname !=
     dottednpath")
 
     >>> localmods = {'foo.__init__': True, 'foo.foo1': True,
     ...              'foo.bar.__init__': True, 'foo.bar.bar1': True,
     ...              'baz.__init__': True, 'baz.baz1': True }
     >>> fromlocal = fromlocalfunc('foo.xxx', localmods)
     >>> # relative
     >>> fromlocal('foo1')
     ('foo.foo1', 'foo.foo1', False)
     >>> fromlocal('bar')
     ('foo.bar', 'foo.bar.__init__', True)
     >>> fromlocal('bar.bar1')
     ('foo.bar.bar1', 'foo.bar.bar1', False)
     >>> # absolute
     >>> fromlocal('baz')
     ('baz', 'baz.__init__', True)
     >>> fromlocal('baz.baz1')
     ('baz.baz1', 'baz.baz1', False)
     >>> # unknown = maybe standard library
     >>> fromlocal('os')
     False
     >>> fromlocal(None, 1)
     ('foo', 'foo.__init__', True)
+    >>> fromlocal('foo1', 1)
+    ('foo.foo1', 'foo.foo1', False)
     >>> fromlocal2 = fromlocalfunc('foo.xxx.yyy', localmods)
     >>> fromlocal2(None, 2)
     ('foo', 'foo.__init__', True)
+    >>> fromlocal2('bar2', 1)
+    False
+    >>> fromlocal2('bar', 2)
+    ('foo.bar', 'foo.bar.__init__', True)
     """
     prefix = '.'.join(modulename.split('.')[:-1])
     if prefix:
         prefix += '.'
     def fromlocal(name, level=0):
-        # name is None when relative imports are used.
-        if name is None:
+        # name is false value when relative imports are used.
+        if not name:
             # If relative imports are used, level must not be absolute.
             assert level > 0
             candidates = ['.'.join(modulename.split('.')[:-level])]
         else:
-            # Check relative name first.
-            candidates = [prefix + name, name]
+            if not level:
+                # Check relative name first.
+                candidates = [prefix + name, name]
+            else:
+                candidates = ['.'.join(modulename.split('.')[:-level]) +
+                              '.' + name]
 
         for n in candidates:
             if n in localmods:
                 return (n, n, False)
             dottedpath = n + '.__init__'
             if dottedpath in localmods:
                 return (n, dottedpath, True)
         return False
     return fromlocal
 
 def list_stdlib_modules():
     """List the modules present in the stdlib.
 
     >>> mods = set(list_stdlib_modules())
     >>> 'BaseHTTPServer' in mods
     True
 
     os.path isn't really a module, so it's missing:
 
     >>> 'os.path' in mods
     False
 
     sys requires special treatment, because it's baked into the
     interpreter, but it should still appear:
 
     >>> 'sys' in mods
     True
 
     >>> 'collections' in mods
     True
 
     >>> 'cStringIO' in mods
     True
+
+    >>> 'cffi' in mods
+    True
     """
     for m in sys.builtin_module_names:
         yield m
     # These modules only exist on windows, but we should always
     # consider them stdlib.
     for m in ['msvcrt', '_winreg']:
         yield m
     yield 'builtins' # python3 only
     for m in 'fcntl', 'grp', 'pwd', 'termios':  # Unix only
         yield m
     for m in 'cPickle', 'datetime': # in Python (not C) on PyPy
         yield m
+    for m in ['cffi']:
+        yield m
     stdlib_prefixes = set([sys.prefix, sys.exec_prefix])
     # We need to supplement the list of prefixes for the search to work
     # when run from within a virtualenv.
     for mod in (BaseHTTPServer, zlib):
         try:
             # Not all module objects have a __file__ attribute.
             filename = mod.__file__
         except AttributeError:
             continue
         dirname = os.path.dirname(filename)
         for prefix in stdlib_prefixes:
             if dirname.startswith(prefix):
                 # Then this directory is redundant.
                 break
         else:
             stdlib_prefixes.add(dirname)
     for libpath in sys.path:
         # We want to walk everything in sys.path that starts with
         # something in stdlib_prefixes.
         if not any(libpath.startswith(p) for p in stdlib_prefixes):
             continue
         for top, dirs, files in os.walk(libpath):
             for i, d in reversed(list(enumerate(dirs))):
                 if (not os.path.exists(os.path.join(top, d, '__init__.py'))
                     or top == libpath and d in ('hgext', 'mercurial')):
                     del dirs[i]
             for name in files:
                 if not name.endswith(('.py', '.so', '.pyc', '.pyo', '.pyd')):
                     continue
                 if name.startswith('__init__.py'):
                     full_path = top
                 else:
                     full_path = os.path.join(top, name)
                 rel_path = full_path[len(libpath) + 1:]
                 mod = dotted_name_of_path(rel_path)
                 yield mod
 
 stdlib_modules = set(list_stdlib_modules())
 
 def imported_modules(source, modulename, f, localmods, ignore_nested=False):
     """Given the source of a file as a string, yield the names
     imported by that file.
 
     Args:
       source: The python source to examine as a string.
       modulename: of specified python source (may have `__init__`)
       localmods: dict of locally defined module names (may have `__init__`)
       ignore_nested: If true, import statements that do not start in
                      column zero will be ignored.
 
     Returns:
       A list of absolute module names imported by the given source.
 
     >>> f = 'foo/xxx.py'
     >>> modulename = 'foo.xxx'
     >>> localmods = {'foo.__init__': True,
     ...              'foo.foo1': True, 'foo.foo2': True,
     ...              'foo.bar.__init__': True, 'foo.bar.bar1': True,
     ...              'baz.__init__': True, 'baz.baz1': True }
     >>> # standard library (= not locally defined ones)
     >>> sorted(imported_modules(
     ...        'from stdlib1 import foo, bar; import stdlib2',
     ...        modulename, f, localmods))
     []
     >>> # relative importing
     >>> sorted(imported_modules(
     ...        'import foo1; from bar import bar1',
     ...        modulename, f, localmods))
     ['foo.bar.bar1', 'foo.foo1']
     >>> sorted(imported_modules(
     ...        'from bar.bar1 import name1, name2, name3',
     ...        modulename, f, localmods))
     ['foo.bar.bar1']
     >>> # absolute importing
     >>> sorted(imported_modules(
     ...        'from baz import baz1, name1',
     ...        modulename, f, localmods))
     ['baz.__init__', 'baz.baz1']
     >>> # mixed importing, even though it shouldn't be recommended
     >>> sorted(imported_modules(
     ...        'import stdlib, foo1, baz',
     ...        modulename, f, localmods))
     ['baz.__init__', 'foo.foo1']
     >>> # ignore_nested
     >>> sorted(imported_modules(
     ... '''import foo
     ... def wat():
     ...     import bar
     ... ''', modulename, f, localmods))
     ['foo.__init__', 'foo.bar.__init__']
     >>> sorted(imported_modules(
     ... '''import foo
     ... def wat():
     ...     import bar
     ... ''', modulename, f, localmods, ignore_nested=True))
     ['foo.__init__']
     """
     fromlocal = fromlocalfunc(modulename, localmods)
     for node in ast.walk(ast.parse(source, f)):
         if ignore_nested and getattr(node, 'col_offset', 0) > 0:
             continue
         if isinstance(node, ast.Import):
             for n in node.names:
                 found = fromlocal(n.name)
                 if not found:
                     # this should import standard library
                     continue
                 yield found[1]
         elif isinstance(node, ast.ImportFrom):
             found = fromlocal(node.module, node.level)
             if not found:
                 # this should import standard library
                 continue
 
             absname, dottedpath, hassubmod = found
             if not hassubmod:
                 # "dottedpath" is not a package; must be imported
                 yield dottedpath
                 # examination of "node.names" should be redundant
                 # e.g.: from mercurial.node import nullid, nullrev
                 continue
 
             modnotfound = False
             prefix = absname + '.'
             for n in node.names:
                 found = fromlocal(prefix + n.name)
                 if not found:
                     # this should be a function or a property of "node.module"
                     modnotfound = True
                     continue
                 yield found[1]
             if modnotfound:
                 # "dottedpath" is a package, but imported because of non-module
                 # lookup
                 yield dottedpath
 
 def verify_import_convention(module, source, localmods):
     """Verify imports match our established coding convention.
 
     We have 2 conventions: legacy and modern. The modern convention is in
     effect when using absolute imports.
 
     The legacy convention only looks for mixed imports. The modern convention
     is much more thorough.
     """
     root = ast.parse(source)
     absolute = usingabsolute(root)
 
     if absolute:
         return verify_modern_convention(module, root, localmods)
     else:
         return verify_stdlib_on_own_line(root)
 
 def verify_modern_convention(module, root, localmods, root_col_offset=0):
     """Verify a file conforms to the modern import convention rules.
 
     The rules of the modern convention are:
 
     * Ordering is stdlib followed by local imports. Each group is lexically
       sorted.
     * Importing multiple modules via "import X, Y" is not allowed: use
       separate import statements.
     * Importing multiple modules via "from X import ..." is allowed if using
       parenthesis and one entry per line.
     * Only 1 relative import statement per import level ("from .", "from ..")
       is allowed.
     * Relative imports from higher levels must occur before lower levels. e.g.
       "from .." must be before "from .".
     * Imports from peer packages should use relative import (e.g. do not
       "import mercurial.foo" from a "mercurial.*" module).
     * Symbols can only be imported from specific modules (see
       `allowsymbolimports`). For other modules, first import the module then
       assign the symbol to a module-level variable. In addition, these imports
-      must be performed before other relative imports. This rule only
+      must be performed before other local imports. This rule only
       applies to import statements outside of any blocks.
     * Relative imports from the standard library are not allowed.
     * Certain modules must be aliased to alternate names to avoid aliasing
       and readability problems. See `requirealias`.
     """
     topmodule = module.split('.')[0]
     fromlocal = fromlocalfunc(module, localmods)
 
     # Whether a local/non-stdlib import has been performed.
     seenlocal = None
-    # Whether a relative, non-symbol import has been seen.
-    seennonsymbolrelative = False
+    # Whether a local/non-stdlib, non-symbol import has been seen.
+    seennonsymbollocal = False
     # The last name to be imported (for sorting).
     lastname = None
     # Relative import levels encountered so far.
     seenlevels = set()
 
     for node, newscope in walklocal(root):
         def msg(fmt, *args):
             return (fmt % args, node.lineno)
         if newscope:
             # Check for local imports in function
             for r in verify_modern_convention(module, node, localmods,
                                               node.col_offset + 4):
                 yield r
         elif isinstance(node, ast.Import):
             # Disallow "import foo, bar" and require separate imports
             # for each module.
             if len(node.names) > 1:
                 yield msg('multiple imported names: %s',
                           ', '.join(n.name for n in node.names))
 
             name = node.names[0].name
             asname = node.names[0].asname
 
             # Ignore sorting rules on imports inside blocks.
             if node.col_offset == root_col_offset:
                 if lastname and name < lastname:
                     yield msg('imports not lexically sorted: %s < %s',
                               name, lastname)
 
                 lastname = name
 
             # stdlib imports should be before local imports.
             stdlib = name in stdlib_modules
             if stdlib and seenlocal and node.col_offset == root_col_offset:
                 yield msg('stdlib import "%s" follows local import: %s',
                           name, seenlocal)
 
             if not stdlib:
                 seenlocal = name
 
             # Import of sibling modules should use relative imports.
             topname = name.split('.')[0]
             if topname == topmodule:
                 yield msg('import should be relative: %s', name)
 
             if name in requirealias and asname != requirealias[name]:
                 yield msg('%s module must be "as" aliased to %s',
                           name, requirealias[name])
 
         elif isinstance(node, ast.ImportFrom):
             # Resolve the full imported module name.
             if node.level > 0:
                 fullname = '.'.join(module.split('.')[:-node.level])
                 if node.module:
                     fullname += '.%s' % node.module
             else:
                 assert node.module
                 fullname = node.module
 
                 topname = fullname.split('.')[0]
                 if topname == topmodule:
                     yield msg('import should be relative: %s', fullname)
 
             # __future__ is special since it needs to come first and use
             # symbol import.
             if fullname != '__future__':
                 if not fullname or fullname in stdlib_modules:
                     yield msg('relative import of stdlib module')
                 else:
                     seenlocal = fullname
 
             # Direct symbol import is only allowed from certain modules and
             # must occur before non-symbol imports.
+            found = fromlocal(node.module, node.level)
+            if found and found[2]:  # node.module is a package
+                prefix = found[0] + '.'
+                symbols = [n.name for n in node.names
+                           if not fromlocal(prefix + n.name)]
+            else:
+                symbols = [n.name for n in node.names]
             if node.module and node.col_offset == root_col_offset:
-                found = fromlocal(node.module, node.level)
-                if found and found[2]:  # node.module is a package
-                    prefix = found[0] + '.'
-                    symbols = [n.name for n in node.names
-                               if not fromlocal(prefix + n.name)]
-                else:
-                    symbols = [n.name for n in node.names]
-
                 if symbols and fullname not in allowsymbolimports:
                     yield msg('direct symbol import %s from %s',
                               ', '.join(symbols), fullname)
 
-                if symbols and seennonsymbolrelative:
+                if symbols and seennonsymbollocal:
                     yield msg('symbol import follows non-symbol import: %s',
                               fullname)
+            if not symbols and fullname not in stdlib_modules:
+                seennonsymbollocal = True
 
             if not node.module:
                 assert node.level
-                seennonsymbolrelative = True
 
                 # Only allow 1 group per level.
                 if (node.level in seenlevels
                     and node.col_offset == root_col_offset):
                     yield msg('multiple "from %s import" statements',
                               '.' * node.level)
 
                 # Higher-level groups come before lower-level groups.
                 if any(node.level > l for l in seenlevels):
                     yield msg('higher-level import should come first: %s',
                               fullname)
 
                 seenlevels.add(node.level)
 
             # Entries in "from .X import ( ... )" lists must be lexically
             # sorted.
             lastentryname = None
 
             for n in node.names:
                 if lastentryname and n.name < lastentryname:
                     yield msg('imports from %s not lexically sorted: %s < %s',
                               fullname, n.name, lastentryname)
 
                 lastentryname = n.name
 
                 if n.name in requirealias and n.asname != requirealias[n.name]:
                     yield msg('%s from %s must be "as" aliased to %s',
                               n.name, fullname, requirealias[n.name])
 
 def verify_stdlib_on_own_line(root):
     """Given some python source, verify that stdlib imports are done
     in separate statements from relative local module imports.
 
     >>> list(verify_stdlib_on_own_line(ast.parse('import sys, foo')))
     [('mixed imports\\n   stdlib:    sys\\n   relative:  foo', 1)]
     >>> list(verify_stdlib_on_own_line(ast.parse('import sys, os')))
     []
     >>> list(verify_stdlib_on_own_line(ast.parse('import foo, bar')))
     []
     """
     for node in ast.walk(root):
         if isinstance(node, ast.Import):
             from_stdlib = {False: [], True: []}
             for n in node.names:
                 from_stdlib[n.name in stdlib_modules].append(n.name)
             if from_stdlib[True] and from_stdlib[False]:
                 yield ('mixed imports\n   stdlib:    %s\n   relative:  %s' %
                        (', '.join(sorted(from_stdlib[True])),
                         ', '.join(sorted(from_stdlib[False]))), node.lineno)
 
 class CircularImport(Exception):
     pass
 
 def checkmod(mod, imports):
     shortest = {}
     visit = [[mod]]
     while visit:
         path = visit.pop(0)
         for i in sorted(imports.get(path[-1], [])):
             if len(path) < shortest.get(i, 1000):
                 shortest[i] = len(path)
                 if i in path:
                     if i == path[0]:
                         raise CircularImport(path)
                     continue
                 visit.append(path + [i])
 
 def rotatecycle(cycle):
     """arrange a cycle so that the lexicographically first module listed first
 
     >>> rotatecycle(['foo', 'bar'])
     ['bar', 'foo', 'bar']
     """
     lowest = min(cycle)
     idx = cycle.index(lowest)
     return cycle[idx:] + cycle[:idx] + [lowest]
 
 def find_cycles(imports):
     """Find cycles in an already-loaded import graph.
 
     All module names recorded in `imports` should be absolute one.
 
     >>> from __future__ import print_function
     >>> imports = {'top.foo': ['top.bar', 'os.path', 'top.qux'],
     ...            'top.bar': ['top.baz', 'sys'],
     ...            'top.baz': ['top.foo'],
     ...            'top.qux': ['top.foo']}
     >>> print('\\n'.join(sorted(find_cycles(imports))))
     top.bar -> top.baz -> top.foo -> top.bar
     top.foo -> top.qux -> top.foo
     """
     cycles = set()
     for mod in sorted(imports.keys()):
         try:
             checkmod(mod, imports)
         except CircularImport as e:
             cycle = e.args[0]
             cycles.add(" -> ".join(rotatecycle(cycle)))
     return cycles
 
 def _cycle_sortkey(c):
     return len(c), c
 
 def embedded(f, modname, src):
     """Extract embedded python code
 
     >>> def test(fn, lines):
     ...     for s, m, f, l in embedded(fn, "example", lines):
     ...         print("%s %s %s" % (m, f, l))
     ...         print(repr(s))
     >>> lines = [
     ...   'comment',
     ...   '  >>> from __future__ import print_function',
     ...   "  >>> ' multiline",
     ...   "  ... string'",
     ...   '  ',
     ...   'comment',
     ...   '  $ cat > foo.py <<EOF',
     ...   '  > from __future__ import print_function',
     ...   '  > EOF',
     ... ]
     >>> test("example.t", lines)
     example[2] doctest.py 2
     "from __future__ import print_function\\n' multiline\\nstring'\\n"
     example[7] foo.py 7
     'from __future__ import print_function\\n'
     """
     inlinepython = 0
     shpython = 0
     script = []
     prefix = 6
     t = ''
     n = 0
     for l in src:
         n += 1
         if not l.endswith(b'\n'):
             l += b'\n'
         if l.startswith(b'  >>> '): # python inlines
             if shpython:
                 print("%s:%d: Parse Error" % (f, n))
             if not inlinepython:
                 # We've just entered a Python block.
                 inlinepython = n
                 t = 'doctest.py'
             script.append(l[prefix:])
             continue
         if l.startswith(b'  ... '): # python inlines
             script.append(l[prefix:])
             continue
         cat = re.search(r"\$ \s*cat\s*>\s*(\S+\.py)\s*<<\s*EOF", l)
         if cat:
             if inlinepython:
                 yield ''.join(script), ("%s[%d]" %
                        (modname, inlinepython)), t, inlinepython
                 script = []
                 inlinepython = 0
             shpython = n
             t = cat.group(1)
             continue
         if shpython and l.startswith(b'  > '): # sh continuation
             if l == b'  > EOF\n':
                 yield ''.join(script), ("%s[%d]" %
                        (modname, shpython)), t, shpython
                 script = []
                 shpython = 0
             else:
                 script.append(l[4:])
             continue
         if inlinepython and l == b'  \n':
             yield ''.join(script), ("%s[%d]" %
                    (modname, inlinepython)), t, inlinepython
             script = []
             inlinepython = 0
             continue
 
 def sources(f, modname):
     """Yields possibly multiple sources from a filepath
 
     input: filepath, modulename
     yields:  script(string), modulename, filepath, linenumber
 
     For embedded scripts, the modulename and filepath will be different
     from the function arguments. linenumber is an offset relative to
     the input file.
     """
     py = False
-    if f.endswith('.py'):
+    if not f.endswith('.t'):
         with open(f) as src:
             yield src.read(), modname, f, 0
             py = True
     if py or f.endswith('.t'):
         with open(f) as src:
             for script, modname, t, line in embedded(f, modname, src):
                 yield script, modname, t, line
 
 def main(argv):
     if len(argv) < 2 or (argv[1] == '-' and len(argv) > 2):
         print('Usage: %s {-|file [file] [file] ...}')
         return 1
     if argv[1] == '-':
         argv = argv[:1]
         argv.extend(l.rstrip() for l in sys.stdin.readlines())
     localmods = {}
     used_imports = {}
     any_errors = False
     for source_path in argv[1:]:
         modname = dotted_name_of_path(source_path, trimpure=True)
         localmods[modname] = source_path
     for localmodname, source_path in sorted(localmods.items()):
         for src, modname, name, line in sources(source_path, localmodname):
             try:
                 used_imports[modname] = sorted(
                     imported_modules(src, modname, name, localmods,
                                      ignore_nested=True))
                 for error, lineno in verify_import_convention(modname, src,
                                                               localmods):
                     any_errors = True
                     print('%s:%d: %s' % (source_path, lineno + line, error))
             except SyntaxError as e:
                 print('%s:%d: SyntaxError: %s' %
                       (source_path, e.lineno + line, e))
     cycles = find_cycles(used_imports)
     if cycles:
         firstmods = set()
         for c in sorted(cycles, key=_cycle_sortkey):
             first = c.split()[0]
             # As a rough cut, ignore any cycle that starts with the
             # same module as some other cycle. Otherwise we see lots
             # of cycles that are effectively duplicates.
             if first in firstmods:
                 continue
             print('Import cycle:', c)
             firstmods.add(first)
         any_errors = True
     return any_errors != 0
 
 if __name__ == '__main__':
     sys.exit(int(main(sys.argv)))

contrib/macosx/Welcome.html

@@ -1,20 +1,20
 <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
-<!-- This is the second screen displayed during the install. -->
+<!-- This is the first screen displayed during the install. -->
 <html>
 <head>
   <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
   <meta http-equiv="Content-Style-Type" content="text/css">
   <title></title>
   <style type="text/css">
     p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; font: 14.0px Helvetica}
     p.p2 {margin: 0.0px 0.0px 0.0px 0.0px; font: 12.0px Helvetica; min-height: 14.0px}
   </style>
 </head>
 <body>
 <p class="p1">This is a prepackaged release of <a href="https://mercurial-scm.org/">Mercurial</a> for Mac OS X.</p>
 <p class="p2"><br></p>
 <br>
 <p>
 Please be sure to read the latest <a href="https://mercurial-scm.org/wiki/WhatsNew">release notes</a>.</p>
 </body>
 </html>

contrib/perf.py

@@ -1,798 +1,902
 # perf.py - performance test routines
 '''helper extension to measure performance'''
 
+# "historical portability" policy of perf.py:
+#
+# We have to do:
+# - make perf.py "loadable" with as wide Mercurial version as possible
+#   This doesn't mean that perf commands work correctly with that Mercurial.
+#   BTW, perf.py itself has been available since 1.1 (or eb240755386d).
+# - make historical perf command work correctly with as wide Mercurial
+#   version as possible
+#
+# We have to do, if possible with reasonable cost:
+# - make recent perf command for historical feature work correctly
+#   with early Mercurial
+#
+# We don't have to do:
+# - make perf command for recent feature work correctly with early
+#   Mercurial
+
 from __future__ import absolute_import
 import functools
 import os
 import random
 import sys
 import time
 from mercurial import (
-    branchmap,
     cmdutil,
     commands,
     copies,
     error,
+    extensions,
     mdiff,
     merge,
-    obsolete,
-    repoview,
     revlog,
-    scmutil,
     util,
 )
 
-formatteropts = commands.formatteropts
-revlogopts = commands.debugrevlogopts
+# for "historical portability":
+# try to import modules separately (in dict order), and ignore
+# failure, because these aren't available with early Mercurial
+try:
+    from mercurial import branchmap # since 2.5 (or bcee63733aad)
+except ImportError:
+    pass
+try:
+    from mercurial import obsolete # since 2.3 (or ad0d6c2b3279)
+except ImportError:
+    pass
+try:
+    from mercurial import repoview # since 2.5 (or 3a6ddacb7198)
+except ImportError:
+    pass
+try:
+    from mercurial import scmutil # since 1.9 (or 8b252e826c68)
+except ImportError:
+    pass
+
+# for "historical portability":
+# define util.safehasattr forcibly, because util.safehasattr has been
+# available since 1.9.3 (or 94b200a11cf7)
+_undefined = object()
+def safehasattr(thing, attr):
+    return getattr(thing, attr, _undefined) is not _undefined
+setattr(util, 'safehasattr', safehasattr)
+
+# for "historical portability":
+# use locally defined empty option list, if formatteropts isn't
+# available, because commands.formatteropts has been available since
+# 3.2 (or 7a7eed5176a4), even though formatting itself has been
+# available since 2.2 (or ae5f92e154d3)
+formatteropts = getattr(commands, "formatteropts", [])
+
+# for "historical portability":
+# use locally defined option list, if debugrevlogopts isn't available,
+# because commands.debugrevlogopts has been available since 3.7 (or
+# 5606f7d0d063), even though cmdutil.openrevlog() has been available
+# since 1.9 (or a79fea6b3e77).
+revlogopts = getattr(commands, "debugrevlogopts", [
+        ('c', 'changelog', False, ('open changelog')),
+        ('m', 'manifest', False, ('open manifest')),
+        ('', 'dir', False, ('open directory manifest')),
+        ])
 
 cmdtable = {}
-command = cmdutil.command(cmdtable)
+
+# for "historical portability":
+# define parsealiases locally, because cmdutil.parsealiases has been
+# available since 1.5 (or 6252852b4332)
+def parsealiases(cmd):
+    return cmd.lstrip("^").split("|")
+
+if safehasattr(cmdutil, 'command'):
+    import inspect
+    command = cmdutil.command(cmdtable)
+    if 'norepo' not in inspect.getargspec(command)[0]:
+        # for "historical portability":
+        # wrap original cmdutil.command, because "norepo" option has
+        # been available since 3.1 (or 75a96326cecb)
+        _command = command
+        def command(name, options=(), synopsis=None, norepo=False):
+            if norepo:
+                commands.norepo += ' %s' % ' '.join(parsealiases(name))
+            return _command(name, list(options), synopsis)
+else:
+    # for "historical portability":
+    # define "@command" annotation locally, because cmdutil.command
+    # has been available since 1.9 (or 2daa5179e73f)
+    def command(name, options=(), synopsis=None, norepo=False):
+        def decorator(func):
+            if synopsis:
+                cmdtable[name] = func, list(options), synopsis
+            else:
+                cmdtable[name] = func, list(options)
+            if norepo:
+                commands.norepo += ' %s' % ' '.join(parsealiases(name))
+            return func
+        return decorator
 
 def getlen(ui):
     if ui.configbool("perf", "stub"):
         return lambda x: 1
     return len
 
 def gettimer(ui, opts=None):
     """return a timer function and formatter: (timer, formatter)
 
     This function exists to gather the creation of formatter in a single
     place instead of duplicating it in all performance commands."""
 
     # enforce an idle period before execution to counteract power management
     # experimental config: perf.presleep
     time.sleep(ui.configint("perf", "presleep", 1))
 
     if opts is None:
         opts = {}
     # redirect all to stderr
     ui = ui.copy()
     ui.fout = ui.ferr
     # get a formatter
     fm = ui.formatter('perf', opts)
     # stub function, runs code only once instead of in a loop
     # experimental config: perf.stub
     if ui.configbool("perf", "stub"):
         return functools.partial(stub_timer, fm), fm
     return functools.partial(_timer, fm), fm
 
 def stub_timer(fm, func, title=None):
     func()
 
 def _timer(fm, func, title=None):
     results = []
     begin = time.time()
     count = 0
     while True:
         ostart = os.times()
         cstart = time.time()
         r = func()
         cstop = time.time()
         ostop = os.times()
         count += 1
         a, b = ostart, ostop
         results.append((cstop - cstart, b[0] - a[0], b[1]-a[1]))
         if cstop - begin > 3 and count >= 100:
             break
         if cstop - begin > 10 and count >= 3:
             break
 
     fm.startitem()
 
     if title:
         fm.write('title', '! %s\n', title)
     if r:
         fm.write('result', '! result: %s\n', r)
     m = min(results)
     fm.plain('!')
     fm.write('wall', ' wall %f', m[0])
     fm.write('comb', ' comb %f', m[1] + m[2])
     fm.write('user', ' user %f', m[1])
     fm.write('sys',  ' sys %f', m[2])
     fm.write('count',  ' (best of %d)', count)
     fm.plain('\n')
 
 @command('perfwalk', formatteropts)
 def perfwalk(ui, repo, *pats, **opts):
     timer, fm = gettimer(ui, opts)
     try:
         m = scmutil.match(repo[None], pats, {})
         timer(lambda: len(list(repo.dirstate.walk(m, [], True, False))))
     except Exception:
         try:
             m = scmutil.match(repo[None], pats, {})
             timer(lambda: len([b for a, b, c in repo.dirstate.statwalk([], m)]))
         except Exception:
             timer(lambda: len(list(cmdutil.walk(repo, pats, {}))))
     fm.end()
 
 @command('perfannotate', formatteropts)
 def perfannotate(ui, repo, f, **opts):
     timer, fm = gettimer(ui, opts)
     fc = repo['.'][f]
     timer(lambda: len(fc.annotate(True)))
     fm.end()
 
 @command('perfstatus',
          [('u', 'unknown', False,
            'ask status to look for unknown files')] + formatteropts)
 def perfstatus(ui, repo, **opts):
     #m = match.always(repo.root, repo.getcwd())
     #timer(lambda: sum(map(len, repo.dirstate.status(m, [], False, False,
     #                                                False))))
     timer, fm = gettimer(ui, opts)
     timer(lambda: sum(map(len, repo.status(unknown=opts['unknown']))))
     fm.end()
 
 @command('perfaddremove', formatteropts)
 def perfaddremove(ui, repo, **opts):
     timer, fm = gettimer(ui, opts)
     try:
         oldquiet = repo.ui.quiet
         repo.ui.quiet = True
         matcher = scmutil.match(repo[None])
         timer(lambda: scmutil.addremove(repo, matcher, "", dry_run=True))
     finally:
         repo.ui.quiet = oldquiet
         fm.end()
 
 def clearcaches(cl):
     # behave somewhat consistently across internal API changes
     if util.safehasattr(cl, 'clearcaches'):
         cl.clearcaches()
     elif util.safehasattr(cl, '_nodecache'):
         from mercurial.node import nullid, nullrev
         cl._nodecache = {nullid: nullrev}
         cl._nodepos = None
 
 @command('perfheads', formatteropts)
 def perfheads(ui, repo, **opts):
     timer, fm = gettimer(ui, opts)
     cl = repo.changelog
     def d():
         len(cl.headrevs())
         clearcaches(cl)
     timer(d)
     fm.end()
 
 @command('perftags', formatteropts)
 def perftags(ui, repo, **opts):
     import mercurial.changelog
     import mercurial.manifest
     timer, fm = gettimer(ui, opts)
     def t():
         repo.changelog = mercurial.changelog.changelog(repo.svfs)
         repo.manifest = mercurial.manifest.manifest(repo.svfs)
         repo._tags = None
         return len(repo.tags())
     timer(t)
     fm.end()
 
 @command('perfancestors', formatteropts)
 def perfancestors(ui, repo, **opts):
     timer, fm = gettimer(ui, opts)
     heads = repo.changelog.headrevs()
     def d():
         for a in repo.changelog.ancestors(heads):
             pass
     timer(d)
     fm.end()
 
 @command('perfancestorset', formatteropts)
 def perfancestorset(ui, repo, revset, **opts):
     timer, fm = gettimer(ui, opts)
     revs = repo.revs(revset)
     heads = repo.changelog.headrevs()
     def d():
         s = repo.changelog.ancestors(heads)
         for rev in revs:
             rev in s
     timer(d)
     fm.end()
 
 @command('perfdirs', formatteropts)
 def perfdirs(ui, repo, **opts):
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     'a' in dirstate
     def d():
         dirstate.dirs()
         del dirstate._dirs
     timer(d)
     fm.end()
 
 @command('perfdirstate', formatteropts)
 def perfdirstate(ui, repo, **opts):
     timer, fm = gettimer(ui, opts)
     "a" in repo.dirstate
     def d():
         repo.dirstate.invalidate()
         "a" in repo.dirstate
     timer(d)
     fm.end()
 
 @command('perfdirstatedirs', formatteropts)
 def perfdirstatedirs(ui, repo, **opts):
     timer, fm = gettimer(ui, opts)
     "a" in repo.dirstate
     def d():
         "a" in repo.dirstate._dirs
         del repo.dirstate._dirs
     timer(d)
     fm.end()
 
 @command('perfdirstatefoldmap', formatteropts)
 def perfdirstatefoldmap(ui, repo, **opts):
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     'a' in dirstate
     def d():
         dirstate._filefoldmap.get('a')
         del dirstate._filefoldmap
     timer(d)
     fm.end()
 
 @command('perfdirfoldmap', formatteropts)
 def perfdirfoldmap(ui, repo, **opts):
     timer, fm = gettimer(ui, opts)
     dirstate = repo.dirstate
     'a' in dirstate
     def d():
         dirstate._dirfoldmap.get('a')
         del dirstate._dirfoldmap
         del dirstate._dirs
     timer(d)
     fm.end()
 
 @command('perfdirstatewrite', formatteropts)
 def perfdirstatewrite(ui, repo, **opts):
     timer, fm = gettimer(ui, opts)
     ds = repo.dirstate
     "a" in ds
     def d():
         ds._dirty = True
         ds.write(repo.currenttransaction())
     timer(d)
     fm.end()
 
 @command('perfmergecalculate',
          [('r', 'rev', '.', 'rev to merge against')] + formatteropts)
 def perfmergecalculate(ui, repo, rev, **opts):
     timer, fm = gettimer(ui, opts)
     wctx = repo[None]
     rctx = scmutil.revsingle(repo, rev, rev)
     ancestor = wctx.ancestor(rctx)
     # we don't want working dir files to be stat'd in the benchmark, so prime
     # that cache
     wctx.dirty()
     def d():
         # acceptremote is True because we don't want prompts in the middle of
         # our benchmark
         merge.calculateupdates(repo, wctx, rctx, [ancestor], False, False,
                                acceptremote=True, followcopies=True)
     timer(d)
     fm.end()
 
 @command('perfpathcopies', [], "REV REV")
 def perfpathcopies(ui, repo, rev1, rev2, **opts):
     timer, fm = gettimer(ui, opts)
     ctx1 = scmutil.revsingle(repo, rev1, rev1)
     ctx2 = scmutil.revsingle(repo, rev2, rev2)
     def d():
         copies.pathcopies(ctx1, ctx2)
     timer(d)
     fm.end()
 
 @command('perfmanifest', [], 'REV')
 def perfmanifest(ui, repo, rev, **opts):
     timer, fm = gettimer(ui, opts)
     ctx = scmutil.revsingle(repo, rev, rev)
     t = ctx.manifestnode()
     def d():
         repo.manifest.clearcaches()
         repo.manifest.read(t)
     timer(d)
     fm.end()
 
 @command('perfchangeset', formatteropts)
 def perfchangeset(ui, repo, rev, **opts):
     timer, fm = gettimer(ui, opts)
     n = repo[rev].node()
     def d():
         repo.changelog.read(n)
         #repo.changelog._cache = None
     timer(d)
     fm.end()
 
 @command('perfindex', formatteropts)
 def perfindex(ui, repo, **opts):
     import mercurial.revlog
     timer, fm = gettimer(ui, opts)
     mercurial.revlog._prereadsize = 2**24 # disable lazy parser in old hg
     n = repo["tip"].node()
     def d():
         cl = mercurial.revlog.revlog(repo.svfs, "00changelog.i")
         cl.rev(n)
     timer(d)
     fm.end()
 
 @command('perfstartup', formatteropts)
 def perfstartup(ui, repo, **opts):
     timer, fm = gettimer(ui, opts)
     cmd = sys.argv[0]
     def d():
         if os.name != 'nt':
             os.system("HGRCPATH= %s version -q > /dev/null" % cmd)
         else:
             os.environ['HGRCPATH'] = ''
             os.system("%s version -q > NUL" % cmd)
     timer(d)
     fm.end()
 
 @command('perfparents', formatteropts)
 def perfparents(ui, repo, **opts):
     timer, fm = gettimer(ui, opts)
     # control the number of commits perfparents iterates over
     # experimental config: perf.parentscount
     count = ui.configint("perf", "parentscount", 1000)
     if len(repo.changelog) < count:
         raise error.Abort("repo needs %d commits for this test" % count)
     repo = repo.unfiltered()
     nl = [repo.changelog.node(i) for i in xrange(count)]
     def d():
         for n in nl:
             repo.changelog.parents(n)
     timer(d)
     fm.end()
 
 @command('perfctxfiles', formatteropts)
 def perfctxfiles(ui, repo, x, **opts):
     x = int(x)
     timer, fm = gettimer(ui, opts)
     def d():
         len(repo[x].files())
     timer(d)
     fm.end()
 
 @command('perfrawfiles', formatteropts)
 def perfrawfiles(ui, repo, x, **opts):
     x = int(x)
     timer, fm = gettimer(ui, opts)
     cl = repo.changelog
     def d():
         len(cl.read(x)[3])
     timer(d)
     fm.end()
 
 @command('perflookup', formatteropts)
 def perflookup(ui, repo, rev, **opts):
     timer, fm = gettimer(ui, opts)
     timer(lambda: len(repo.lookup(rev)))
     fm.end()
 
 @command('perfrevrange', formatteropts)
 def perfrevrange(ui, repo, *specs, **opts):
     timer, fm = gettimer(ui, opts)
     revrange = scmutil.revrange
     timer(lambda: len(revrange(repo, specs)))
     fm.end()
 
 @command('perfnodelookup', formatteropts)
 def perfnodelookup(ui, repo, rev, **opts):
     timer, fm = gettimer(ui, opts)
     import mercurial.revlog
     mercurial.revlog._prereadsize = 2**24 # disable lazy parser in old hg
     n = repo[rev].node()
     cl = mercurial.revlog.revlog(repo.svfs, "00changelog.i")
     def d():
         cl.rev(n)
         clearcaches(cl)
     timer(d)
     fm.end()
 
 @command('perflog',
          [('', 'rename', False, 'ask log to follow renames')] + formatteropts)
 def perflog(ui, repo, rev=None, **opts):
     if rev is None:
         rev=[]
     timer, fm = gettimer(ui, opts)
     ui.pushbuffer()
     timer(lambda: commands.log(ui, repo, rev=rev, date='', user='',
                                copies=opts.get('rename')))
     ui.popbuffer()
     fm.end()
 
 @command('perfmoonwalk', formatteropts)
 def perfmoonwalk(ui, repo, **opts):
     """benchmark walking the changelog backwards
 
     This also loads the changelog data for each revision in the changelog.
     """
     timer, fm = gettimer(ui, opts)
     def moonwalk():
         for i in xrange(len(repo), -1, -1):
             ctx = repo[i]
             ctx.branch() # read changelog data (in addition to the index)
     timer(moonwalk)
     fm.end()
 
 @command('perftemplating', formatteropts)
 def perftemplating(ui, repo, rev=None, **opts):
     if rev is None:
         rev=[]
     timer, fm = gettimer(ui, opts)
     ui.pushbuffer()
     timer(lambda: commands.log(ui, repo, rev=rev, date='', user='',
                                template='{date|shortdate} [{rev}:{node|short}]'
                                ' {author|person}: {desc|firstline}\n'))
     ui.popbuffer()
     fm.end()
 
 @command('perfcca', formatteropts)
 def perfcca(ui, repo, **opts):
     timer, fm = gettimer(ui, opts)
     timer(lambda: scmutil.casecollisionauditor(ui, False, repo.dirstate))
     fm.end()
 
 @command('perffncacheload', formatteropts)
 def perffncacheload(ui, repo, **opts):
     timer, fm = gettimer(ui, opts)
     s = repo.store
     def d():
         s.fncache._load()
     timer(d)
     fm.end()
 
 @command('perffncachewrite', formatteropts)
 def perffncachewrite(ui, repo, **opts):
     timer, fm = gettimer(ui, opts)
     s = repo.store
     s.fncache._load()
     lock = repo.lock()
     tr = repo.transaction('perffncachewrite')
     def d():
         s.fncache._dirty = True
         s.fncache.write(tr)
     timer(d)
     lock.release()
     tr.close()
     fm.end()
 
 @command('perffncacheencode', formatteropts)
 def perffncacheencode(ui, repo, **opts):
     timer, fm = gettimer(ui, opts)
     s = repo.store
     s.fncache._load()
     def d():
         for p in s.fncache.entries:
             s.encode(p)
     timer(d)
     fm.end()
 
 @command('perfdiffwd', formatteropts)
 def perfdiffwd(ui, repo, **opts):
     """Profile diff of working directory changes"""
     timer, fm = gettimer(ui, opts)
     options = {
         'w': 'ignore_all_space',
         'b': 'ignore_space_change',
         'B': 'ignore_blank_lines',
         }
 
     for diffopt in ('', 'w', 'b', 'B', 'wB'):
         opts = dict((options[c], '1') for c in diffopt)
         def d():
             ui.pushbuffer()
             commands.diff(ui, repo, **opts)
             ui.popbuffer()
         title = 'diffopts: %s' % (diffopt and ('-' + diffopt) or 'none')
         timer(d, title)
     fm.end()
 
 @command('perfrevlog', revlogopts + formatteropts +
          [('d', 'dist', 100, 'distance between the revisions'),
           ('s', 'startrev', 0, 'revision to start reading at')],
          '-c|-m|FILE')
 def perfrevlog(ui, repo, file_=None, startrev=0, **opts):
     """Benchmark reading a series of revisions from a revlog.
 
     By default, we read every ``-d/--dist`` revision from 0 to tip of
     the specified revlog.
 
     The start revision can be defined via ``-s/--startrev``.
     """
     timer, fm = gettimer(ui, opts)
     dist = opts['dist']
     _len = getlen(ui)
     def d():
         r = cmdutil.openrevlog(repo, 'perfrevlog', file_, opts)
         for x in xrange(startrev, _len(r), dist):
             r.revision(r.node(x))
 
     timer(d)
     fm.end()
 
 @command('perfrevlogrevision', revlogopts + formatteropts +
          [('', 'cache', False, 'use caches instead of clearing')],
          '-c|-m|FILE REV')
 def perfrevlogrevision(ui, repo, file_, rev=None, cache=None, **opts):
     """Benchmark obtaining a revlog revision.
 
     Obtaining a revlog revision consists of roughly the following steps:
 
     1. Compute the delta chain
     2. Obtain the raw chunks for that delta chain
     3. Decompress each raw chunk
     4. Apply binary patches to obtain fulltext
     5. Verify hash of fulltext
 
     This command measures the time spent in each of these phases.
     """
     if opts.get('changelog') or opts.get('manifest'):
         file_, rev = None, file_
     elif rev is None:
         raise error.CommandError('perfrevlogrevision', 'invalid arguments')
 
     r = cmdutil.openrevlog(repo, 'perfrevlogrevision', file_, opts)
     node = r.lookup(rev)
     rev = r.rev(node)
 
     def dodeltachain(rev):
         if not cache:
             r.clearcaches()
         r._deltachain(rev)
 
     def doread(chain):
         if not cache:
             r.clearcaches()
         r._chunkraw(chain[0], chain[-1])
 
     def dodecompress(data, chain):
         if not cache:
             r.clearcaches()
 
         start = r.start
         length = r.length
         inline = r._inline
         iosize = r._io.size
         buffer = util.buffer
         offset = start(chain[0])
 
         for rev in chain:
             chunkstart = start(rev)
             if inline:
                 chunkstart += (rev + 1) * iosize
             chunklength = length(rev)
             b = buffer(data, chunkstart - offset, chunklength)
             revlog.decompress(b)
 
     def dopatch(text, bins):
         if not cache:
             r.clearcaches()
         mdiff.patches(text, bins)
 
     def dohash(text):
         if not cache:
             r.clearcaches()
         r._checkhash(text, node, rev)
 
     def dorevision():
         if not cache:
             r.clearcaches()
         r.revision(node)
 
     chain = r._deltachain(rev)[0]
     data = r._chunkraw(chain[0], chain[-1])[1]
     bins = r._chunks(chain)
     text = str(bins[0])
     bins = bins[1:]
     text = mdiff.patches(text, bins)
 
     benches = [
         (lambda: dorevision(), 'full'),
         (lambda: dodeltachain(rev), 'deltachain'),
         (lambda: doread(chain), 'read'),
         (lambda: dodecompress(data, chain), 'decompress'),
         (lambda: dopatch(text, bins), 'patch'),
         (lambda: dohash(text), 'hash'),
     ]
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
 
 @command('perfrevset',
          [('C', 'clear', False, 'clear volatile cache between each call.'),
           ('', 'contexts', False, 'obtain changectx for each revision')]
          + formatteropts, "REVSET")
 def perfrevset(ui, repo, expr, clear=False, contexts=False, **opts):
     """benchmark the execution time of a revset
 
     Use the --clean option if need to evaluate the impact of build volatile
     revisions set cache on the revset execution. Volatile cache hold filtered
     and obsolete related cache."""
     timer, fm = gettimer(ui, opts)
     def d():
         if clear:
             repo.invalidatevolatilesets()
         if contexts:
             for ctx in repo.set(expr): pass
         else:
             for r in repo.revs(expr): pass
     timer(d)
     fm.end()
 
 @command('perfvolatilesets', formatteropts)
 def perfvolatilesets(ui, repo, *names, **opts):
     """benchmark the computation of various volatile set
 
     Volatile set computes element related to filtering and obsolescence."""
     timer, fm = gettimer(ui, opts)
     repo = repo.unfiltered()
 
     def getobs(name):
         def d():
             repo.invalidatevolatilesets()
             obsolete.getrevs(repo, name)
         return d
 
     allobs = sorted(obsolete.cachefuncs)
     if names:
         allobs = [n for n in allobs if n in names]
 
     for name in allobs:
         timer(getobs(name), title=name)
 
     def getfiltered(name):
         def d():
             repo.invalidatevolatilesets()
             repoview.filterrevs(repo, name)
         return d
 
     allfilter = sorted(repoview.filtertable)
     if names:
         allfilter = [n for n in allfilter if n in names]
 
     for name in allfilter:
         timer(getfiltered(name), title=name)
     fm.end()
 
 @command('perfbranchmap',
          [('f', 'full', False,
            'Includes build time of subset'),
          ] + formatteropts)
 def perfbranchmap(ui, repo, full=False, **opts):
     """benchmark the update of a branchmap
 
     This benchmarks the full repo.branchmap() call with read and write disabled
     """
     timer, fm = gettimer(ui, opts)
     def getbranchmap(filtername):
         """generate a benchmark function for the filtername"""
         if filtername is None:
             view = repo
         else:
             view = repo.filtered(filtername)
         def d():
             if full:
                 view._branchcaches.clear()
             else:
                 view._branchcaches.pop(filtername, None)
             view.branchmap()
         return d
     # add filter in smaller subset to bigger subset
     possiblefilters = set(repoview.filtertable)
     allfilters = []
     while possiblefilters:
         for name in possiblefilters:
             subset = branchmap.subsettable.get(name)
             if subset not in possiblefilters:
                 break
         else:
             assert False, 'subset cycle %s!' % possiblefilters
         allfilters.append(name)
         possiblefilters.remove(name)
 
     # warm the cache
     if not full:
         for name in allfilters:
             repo.filtered(name).branchmap()
     # add unfiltered
     allfilters.append(None)
     oldread = branchmap.read
     oldwrite = branchmap.branchcache.write
     try:
         branchmap.read = lambda repo: None
         branchmap.write = lambda repo: None
         for name in allfilters:
             timer(getbranchmap(name), title=str(name))
     finally:
         branchmap.read = oldread
         branchmap.branchcache.write = oldwrite
     fm.end()
 
 @command('perfloadmarkers')
 def perfloadmarkers(ui, repo):
     """benchmark the time to parse the on-disk markers for a repo
 
     Result is the number of markers in the repo."""
     timer, fm = gettimer(ui)
     timer(lambda: len(obsolete.obsstore(repo.svfs)))
     fm.end()
 
 @command('perflrucachedict', formatteropts +
     [('', 'size', 4, 'size of cache'),
      ('', 'gets', 10000, 'number of key lookups'),
      ('', 'sets', 10000, 'number of key sets'),
      ('', 'mixed', 10000, 'number of mixed mode operations'),
      ('', 'mixedgetfreq', 50, 'frequency of get vs set ops in mixed mode')],
     norepo=True)
 def perflrucache(ui, size=4, gets=10000, sets=10000, mixed=10000,
                  mixedgetfreq=50, **opts):
     def doinit():
         for i in xrange(10000):
             util.lrucachedict(size)
 
     values = []
     for i in xrange(size):
         values.append(random.randint(0, sys.maxint))
 
     # Get mode fills the cache and tests raw lookup performance with no
     # eviction.
     getseq = []
     for i in xrange(gets):
         getseq.append(random.choice(values))
 
     def dogets():
         d = util.lrucachedict(size)
         for v in values:
             d[v] = v
         for key in getseq:
             value = d[key]
             value # silence pyflakes warning
 
     # Set mode tests insertion speed with cache eviction.
     setseq = []
     for i in xrange(sets):
         setseq.append(random.randint(0, sys.maxint))
 
     def dosets():
         d = util.lrucachedict(size)
         for v in setseq:
             d[v] = v
 
     # Mixed mode randomly performs gets and sets with eviction.
     mixedops = []
     for i in xrange(mixed):
         r = random.randint(0, 100)
         if r < mixedgetfreq:
             op = 0
         else:
             op = 1
 
         mixedops.append((op, random.randint(0, size * 2)))
 
     def domixed():
         d = util.lrucachedict(size)
 
         for op, v in mixedops:
             if op == 0:
                 try:
                     d[v]
                 except KeyError:
                     pass
             else:
                 d[v] = v
 
     benches = [
         (doinit, 'init'),
         (dogets, 'gets'),
         (dosets, 'sets'),
         (domixed, 'mixed')
     ]
 
     for fn, title in benches:
         timer, fm = gettimer(ui, opts)
         timer(fn, title=title)
         fm.end()
+
+def uisetup(ui):
+    if (util.safehasattr(cmdutil, 'openrevlog') and
+        not util.safehasattr(commands, 'debugrevlogopts')):
+        # for "historical portability":
+        # In this case, Mercurial should be 1.9 (or a79fea6b3e77) -
+        # 3.7 (or 5606f7d0d063). Therefore, '--dir' option for
+        # openrevlog() should cause failure, because it has been
+        # available since 3.5 (or 49c583ca48c4).
+        def openrevlog(orig, repo, cmd, file_, opts):
+            if opts.get('dir') and not util.safehasattr(repo, 'dirlog'):
+                raise error.Abort("This version doesn't support --dir option",
+                                  hint="use 3.5 or later")
+            return orig(repo, cmd, file_, opts)
+        extensions.wrapfunction(cmdutil, 'openrevlog', openrevlog)

contrib/revsetbenchmarks.py

@@ -1,334 +1,325
 #!/usr/bin/env python
 
 # Measure the performance of a list of revsets against multiple revisions
 # defined by parameter. Checkout one by one and run perfrevset with every
 # revset in the list to benchmark its performance.
 #
 # You should run this from the root of your mercurial repository.
 #
 # call with --help for details
 
 from __future__ import absolute_import, print_function
 import math
+import optparse  # cannot use argparse, python 2.7 only
 import os
 import re
+import subprocess
 import sys
-from subprocess import (
-    CalledProcessError,
-    check_call,
-    PIPE,
-    Popen,
-    STDOUT,
-)
-# cannot use argparse, python 2.7 only
-from optparse import (
-    OptionParser,
-)
 
 DEFAULTVARIANTS = ['plain', 'min', 'max', 'first', 'last',
                    'reverse', 'reverse+first', 'reverse+last',
                    'sort', 'sort+first', 'sort+last']
 
 def check_output(*args, **kwargs):
-    kwargs.setdefault('stderr', PIPE)
-    kwargs.setdefault('stdout', PIPE)
-    proc = Popen(*args, **kwargs)
+    kwargs.setdefault('stderr', subprocess.PIPE)
+    kwargs.setdefault('stdout', subprocess.PIPE)
+    proc = subprocess.Popen(*args, **kwargs)
     output, error = proc.communicate()
     if proc.returncode != 0:
-        raise CalledProcessError(proc.returncode, ' '.join(args[0]))
+        raise subprocess.CalledProcessError(proc.returncode, ' '.join(args[0]))
     return output
 
 def update(rev):
     """update the repo to a revision"""
     try:
-        check_call(['hg', 'update', '--quiet', '--check', str(rev)])
+        subprocess.check_call(['hg', 'update', '--quiet', '--check', str(rev)])
         check_output(['make', 'local'],
                      stderr=None)  # suppress output except for error/warning
-    except CalledProcessError as exc:
+    except subprocess.CalledProcessError as exc:
         print('update to revision %s failed, aborting'%rev, file=sys.stderr)
         sys.exit(exc.returncode)
 
 
 def hg(cmd, repo=None):
     """run a mercurial command
 
     <cmd> is the list of command + argument,
     <repo> is an optional repository path to run this command in."""
     fullcmd = ['./hg']
     if repo is not None:
         fullcmd += ['-R', repo]
     fullcmd += ['--config',
                 'extensions.perf=' + os.path.join(contribdir, 'perf.py')]
     fullcmd += cmd
-    return check_output(fullcmd, stderr=STDOUT)
+    return check_output(fullcmd, stderr=subprocess.STDOUT)
 
 def perf(revset, target=None, contexts=False):
     """run benchmark for this very revset"""
     try:
         args = ['perfrevset', revset]
         if contexts:
             args.append('--contexts')
         output = hg(args, repo=target)
         return parseoutput(output)
-    except CalledProcessError as exc:
+    except subprocess.CalledProcessError as exc:
         print('abort: cannot run revset benchmark: %s'%exc.cmd, file=sys.stderr)
         if getattr(exc, 'output', None) is None: # no output before 2.7
             print('(no output)', file=sys.stderr)
         else:
             print(exc.output, file=sys.stderr)
         return None
 
 outputre = re.compile(r'! wall (\d+.\d+) comb (\d+.\d+) user (\d+.\d+) '
                       'sys (\d+.\d+) \(best of (\d+)\)')
 
 def parseoutput(output):
     """parse a textual output into a dict
 
     We cannot just use json because we want to compare with old
     versions of Mercurial that may not support json output.
     """
     match = outputre.search(output)
     if not match:
         print('abort: invalid output:', file=sys.stderr)
         print(output, file=sys.stderr)
         sys.exit(1)
     return {'comb': float(match.group(2)),
             'count': int(match.group(5)),
             'sys': float(match.group(3)),
             'user': float(match.group(4)),
             'wall': float(match.group(1)),
             }
 
 def printrevision(rev):
     """print data about a revision"""
     sys.stdout.write("Revision ")
     sys.stdout.flush()
-    check_call(['hg', 'log', '--rev', str(rev), '--template',
-                '{if(tags, " ({tags})")} '
-                '{rev}:{node|short}: {desc|firstline}\n'])
+    subprocess.check_call(['hg', 'log', '--rev', str(rev), '--template',
+                           '{if(tags, " ({tags})")} '
+                           '{rev}:{node|short}: {desc|firstline}\n'])
 
 def idxwidth(nbidx):
     """return the max width of number used for index
 
     This is similar to log10(nbidx), but we use custom code here
     because we start with zero and we'd rather not deal with all the
     extra rounding business that log10 would imply.
     """
     nbidx -= 1 # starts at 0
     idxwidth = 0
     while nbidx:
         idxwidth += 1
         nbidx //= 10
     if not idxwidth:
         idxwidth = 1
     return idxwidth
 
 def getfactor(main, other, field, sensitivity=0.05):
     """return the relative factor between values for 'field' in main and other
 
     Return None if the factor is insignificant (less than <sensitivity>
     variation)."""
     factor = 1
     if main is not None:
         factor = other[field] / main[field]
     low, high = 1 - sensitivity, 1 + sensitivity
     if (low < factor < high):
         return None
     return factor
 
 def formatfactor(factor):
     """format a factor into a 4 char string
 
      22%
     156%
     x2.4
      x23
     x789
     x1e4
     x5x7
 
     """
     if factor is None:
         return '    '
     elif factor < 2:
         return '%3i%%' % (factor * 100)
     elif factor < 10:
         return 'x%3.1f' % factor
     elif factor < 1000:
         return '%4s' % ('x%i' % factor)
     else:
         order = int(math.log(factor)) + 1
         while 1 < math.log(factor):
             factor //= 0
         return 'x%ix%i' % (factor, order)
 
 def formattiming(value):
     """format a value to strictly 8 char, dropping some precision if needed"""
     if value < 10**7:
         return ('%.6f' % value)[:8]
     else:
         # value is HUGE very unlikely to happen (4+ month run)
         return '%i' % value
 
 _marker = object()
 def printresult(variants, idx, data, maxidx, verbose=False, reference=_marker):
     """print a line of result to stdout"""
     mask = '%%0%ii) %%s' % idxwidth(maxidx)
 
     out = []
     for var in variants:
         if data[var] is None:
             out.append('error   ')
             out.append(' ' * 4)
             continue
         out.append(formattiming(data[var]['wall']))
         if reference is not _marker:
             factor = None
             if reference is not None:
                 factor = getfactor(reference[var], data[var], 'wall')
             out.append(formatfactor(factor))
         if verbose:
             out.append(formattiming(data[var]['comb']))
             out.append(formattiming(data[var]['user']))
             out.append(formattiming(data[var]['sys']))
             out.append('%6d'    % data[var]['count'])
     print(mask % (idx, ' '.join(out)))
 
 def printheader(variants, maxidx, verbose=False, relative=False):
     header = [' ' * (idxwidth(maxidx) + 1)]
     for var in variants:
         if not var:
             var = 'iter'
         if 8 < len(var):
             var = var[:3] + '..' + var[-3:]
         header.append('%-8s' % var)
         if relative:
             header.append('    ')
         if verbose:
             header.append('%-8s' % 'comb')
             header.append('%-8s' % 'user')
             header.append('%-8s' % 'sys')
             header.append('%6s' % 'count')
     print(' '.join(header))
 
 def getrevs(spec):
     """get the list of rev matched by a revset"""
     try:
         out = check_output(['hg', 'log', '--template={rev}\n', '--rev', spec])
-    except CalledProcessError as exc:
+    except subprocess.CalledProcessError as exc:
         print("abort, can't get revision from %s"%spec, file=sys.stderr)
         sys.exit(exc.returncode)
     return [r for r in out.split() if r]
 
 
 def applyvariants(revset, variant):
     if variant == 'plain':
         return revset
     for var in variant.split('+'):
         revset = '%s(%s)' % (var, revset)
     return revset
 
 helptext="""This script will run multiple variants of provided revsets using
 different revisions in your mercurial repository. After the benchmark are run
 summary output is provided. Use it to demonstrate speed improvements or pin
 point regressions. Revsets to run are specified in a file (or from stdin), one
 revsets per line. Line starting with '#' will be ignored, allowing insertion of
 comments."""
-parser = OptionParser(usage="usage: %prog [options] <revs>",
-                      description=helptext)
+parser = optparse.OptionParser(usage="usage: %prog [options] <revs>",
+                               description=helptext)
 parser.add_option("-f", "--file",
                   help="read revset from FILE (stdin if omitted)",
                   metavar="FILE")
 parser.add_option("-R", "--repo",
                   help="run benchmark on REPO", metavar="REPO")
 
 parser.add_option("-v", "--verbose",
                   action='store_true',
                   help="display all timing data (not just best total time)")
 
 parser.add_option("", "--variants",
                   default=','.join(DEFAULTVARIANTS),
                   help="comma separated list of variant to test "
                        "(eg: plain,min,sorted) (plain = no modification)")
 parser.add_option('', '--contexts',
                   action='store_true',
                   help='obtain changectx from results instead of integer revs')
 
 (options, args) = parser.parse_args()
 
 if not args:
     parser.print_help()
     sys.exit(255)
 
 # the directory where both this script and the perf.py extension live.
 contribdir = os.path.dirname(__file__)
 
 revsetsfile = sys.stdin
 if options.file:
     revsetsfile = open(options.file)
 
 revsets = [l.strip() for l in revsetsfile if not l.startswith('#')]
 revsets = [l for l in revsets if l]
 
 print("Revsets to benchmark")
 print("----------------------------")
 
 for idx, rset in enumerate(revsets):
     print("%i) %s" % (idx, rset))
 
 print("----------------------------")
 print()
 
 revs = []
 for a in args:
     revs.extend(getrevs(a))
 
 variants = options.variants.split(',')
 
 results = []
 for r in revs:
     print("----------------------------")
     printrevision(r)
     print("----------------------------")
     update(r)
     res = []
     results.append(res)
     printheader(variants, len(revsets), verbose=options.verbose)
     for idx, rset in enumerate(revsets):
         varres = {}
         for var in variants:
             varrset = applyvariants(rset, var)
             data = perf(varrset, target=options.repo, contexts=options.contexts)
             varres[var] = data
         res.append(varres)
         printresult(variants, idx, varres, len(revsets),
                     verbose=options.verbose)
         sys.stdout.flush()
     print("----------------------------")
 
 
 print("""
 
 Result by revset
 ================
 """)
 
 print('Revision:')
 for idx, rev in enumerate(revs):
     sys.stdout.write('%i) ' % idx)
     sys.stdout.flush()
     printrevision(rev)
 
 print()
 print()
 
 for ridx, rset in enumerate(revsets):
 
     print("revset #%i: %s" % (ridx, rset))
     printheader(variants, len(results), verbose=options.verbose, relative=True)
     ref = None
     for idx, data in enumerate(results):
         printresult(variants, idx, data[ridx], len(results),
                     verbose=options.verbose, reference=ref)
         ref = data[ridx]
     print()

contrib/synthrepo.py

@@ -1,515 +1,516
 # synthrepo.py - repo synthesis
 #
 # Copyright 2012 Facebook
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''synthesize structurally interesting change history
 
 This extension is useful for creating a repository with properties
 that are statistically similar to an existing repository. During
 analysis, a simple probability table is constructed from the history
 of an existing repository.  During synthesis, these properties are
 reconstructed.
 
 Properties that are analyzed and synthesized include the following:
 
 - Lines added or removed when an existing file is modified
 - Number and sizes of files added
 - Number of files removed
 - Line lengths
 - Topological distance to parent changeset(s)
 - Probability of a commit being a merge
 - Probability of a newly added file being added to a new directory
 - Interarrival time, and time zone, of commits
 - Number of files in each directory
 
 A few obvious properties that are not currently handled realistically:
 
 - Merges are treated as regular commits with two parents, which is not
   realistic
 - Modifications are not treated as operations on hunks of lines, but
   as insertions and deletions of randomly chosen single lines
 - Committer ID (always random)
 - Executability of files
 - Symlinks and binary files are ignored
 '''
 
 from __future__ import absolute_import
 import bisect
 import collections
 import itertools
 import json
 import os
 import random
 import sys
 import time
+
+from mercurial.i18n import _
+from mercurial.node import (
+    nullid,
+    nullrev,
+    short,
+)
 from mercurial import (
     cmdutil,
     context,
     error,
     hg,
     patch,
     scmutil,
     util,
 )
-from mercurial.i18n import _
-from mercurial.node import (
-    nullid,
-    nullrev,
-    short,
-)
 
 # Note for extension authors: ONLY specify testedwith = 'internal' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'internal'
 
 cmdtable = {}
 command = cmdutil.command(cmdtable)
 
 newfile = set(('new fi', 'rename', 'copy f', 'copy t'))
 
 def zerodict():
     return collections.defaultdict(lambda: 0)
 
 def roundto(x, k):
     if x > k * 2:
         return int(round(x / float(k)) * k)
     return int(round(x))
 
 def parsegitdiff(lines):
     filename, mar, lineadd, lineremove = None, None, zerodict(), 0
     binary = False
     for line in lines:
         start = line[:6]
         if start == 'diff -':
             if filename:
                 yield filename, mar, lineadd, lineremove, binary
             mar, lineadd, lineremove, binary = 'm', zerodict(), 0, False
             filename = patch.gitre.match(line).group(1)
         elif start in newfile:
             mar = 'a'
         elif start == 'GIT bi':
             binary = True
         elif start == 'delete':
             mar = 'r'
         elif start:
             s = start[0]
             if s == '-' and not line.startswith('--- '):
                 lineremove += 1
             elif s == '+' and not line.startswith('+++ '):
                 lineadd[roundto(len(line) - 1, 5)] += 1
     if filename:
         yield filename, mar, lineadd, lineremove, binary
 
 @command('analyze',
          [('o', 'output', '', _('write output to given file'), _('FILE')),
           ('r', 'rev', [], _('analyze specified revisions'), _('REV'))],
          _('hg analyze'), optionalrepo=True)
 def analyze(ui, repo, *revs, **opts):
     '''create a simple model of a repository to use for later synthesis
 
     This command examines every changeset in the given range (or all
     of history if none are specified) and creates a simple statistical
     model of the history of the repository. It also measures the directory
     structure of the repository as checked out.
 
     The model is written out to a JSON file, and can be used by
     :hg:`synthesize` to create or augment a repository with synthetic
     commits that have a structure that is statistically similar to the
     analyzed repository.
     '''
     root = repo.root
     if not root.endswith(os.path.sep):
         root += os.path.sep
 
     revs = list(revs)
     revs.extend(opts['rev'])
     if not revs:
         revs = [':']
 
     output = opts['output']
     if not output:
         output = os.path.basename(root) + '.json'
 
     if output == '-':
         fp = sys.stdout
     else:
         fp = open(output, 'w')
 
     # Always obtain file counts of each directory in the given root directory.
     def onerror(e):
         ui.warn(_('error walking directory structure: %s\n') % e)
 
     dirs = {}
     rootprefixlen = len(root)
     for dirpath, dirnames, filenames in os.walk(root, onerror=onerror):
         dirpathfromroot = dirpath[rootprefixlen:]
         dirs[dirpathfromroot] = len(filenames)
         if '.hg' in dirnames:
             dirnames.remove('.hg')
 
     lineschanged = zerodict()
     children = zerodict()
     p1distance = zerodict()
     p2distance = zerodict()
     linesinfilesadded = zerodict()
     fileschanged = zerodict()
     filesadded = zerodict()
     filesremoved = zerodict()
     linelengths = zerodict()
     interarrival = zerodict()
     parents = zerodict()
     dirsadded = zerodict()
     tzoffset = zerodict()
 
     # If a mercurial repo is available, also model the commit history.
     if repo:
         revs = scmutil.revrange(repo, revs)
         revs.sort()
 
         progress = ui.progress
         _analyzing = _('analyzing')
         _changesets = _('changesets')
         _total = len(revs)
 
         for i, rev in enumerate(revs):
             progress(_analyzing, i, unit=_changesets, total=_total)
             ctx = repo[rev]
             pl = ctx.parents()
             pctx = pl[0]
             prev = pctx.rev()
             children[prev] += 1
             p1distance[rev - prev] += 1
             parents[len(pl)] += 1
             tzoffset[ctx.date()[1]] += 1
             if len(pl) > 1:
                 p2distance[rev - pl[1].rev()] += 1
             if prev == rev - 1:
                 lastctx = pctx
             else:
                 lastctx = repo[rev - 1]
             if lastctx.rev() != nullrev:
                 timedelta = ctx.date()[0] - lastctx.date()[0]
                 interarrival[roundto(timedelta, 300)] += 1
             diff = sum((d.splitlines() for d in ctx.diff(pctx, git=True)), [])
             fileadds, diradds, fileremoves, filechanges = 0, 0, 0, 0
             for filename, mar, lineadd, lineremove, isbin in parsegitdiff(diff):
                 if isbin:
                     continue
                 added = sum(lineadd.itervalues(), 0)
                 if mar == 'm':
                     if added and lineremove:
                         lineschanged[roundto(added, 5),
                                      roundto(lineremove, 5)] += 1
                         filechanges += 1
                 elif mar == 'a':
                     fileadds += 1
                     if '/' in filename:
                         filedir = filename.rsplit('/', 1)[0]
                         if filedir not in pctx.dirs():
                             diradds += 1
                     linesinfilesadded[roundto(added, 5)] += 1
                 elif mar == 'r':
                     fileremoves += 1
                 for length, count in lineadd.iteritems():
                     linelengths[length] += count
             fileschanged[filechanges] += 1
             filesadded[fileadds] += 1
             dirsadded[diradds] += 1
             filesremoved[fileremoves] += 1
 
     invchildren = zerodict()
 
     for rev, count in children.iteritems():
         invchildren[count] += 1
 
     if output != '-':
         ui.status(_('writing output to %s\n') % output)
 
     def pronk(d):
         return sorted(d.iteritems(), key=lambda x: x[1], reverse=True)
 
     json.dump({'revs': len(revs),
                'initdirs': pronk(dirs),
                'lineschanged': pronk(lineschanged),
                'children': pronk(invchildren),
                'fileschanged': pronk(fileschanged),
                'filesadded': pronk(filesadded),
                'linesinfilesadded': pronk(linesinfilesadded),
                'dirsadded': pronk(dirsadded),
                'filesremoved': pronk(filesremoved),
                'linelengths': pronk(linelengths),
                'parents': pronk(parents),
                'p1distance': pronk(p1distance),
                'p2distance': pronk(p2distance),
                'interarrival': pronk(interarrival),
                'tzoffset': pronk(tzoffset),
                },
               fp)
     fp.close()
 
 @command('synthesize',
          [('c', 'count', 0, _('create given number of commits'), _('COUNT')),
           ('', 'dict', '', _('path to a dictionary of words'), _('FILE')),
           ('', 'initfiles', 0, _('initial file count to create'), _('COUNT'))],
          _('hg synthesize [OPTION].. DESCFILE'))
 def synthesize(ui, repo, descpath, **opts):
     '''synthesize commits based on a model of an existing repository
 
     The model must have been generated by :hg:`analyze`. Commits will
     be generated randomly according to the probabilities described in
     the model. If --initfiles is set, the repository will be seeded with
     the given number files following the modeled repository's directory
     structure.
 
     When synthesizing new content, commit descriptions, and user
     names, words will be chosen randomly from a dictionary that is
     presumed to contain one word per line. Use --dict to specify the
     path to an alternate dictionary to use.
     '''
     try:
         fp = hg.openpath(ui, descpath)
     except Exception as err:
         raise error.Abort('%s: %s' % (descpath, err[0].strerror))
     desc = json.load(fp)
     fp.close()
 
     def cdf(l):
         if not l:
             return [], []
         vals, probs = zip(*sorted(l, key=lambda x: x[1], reverse=True))
         t = float(sum(probs, 0))
         s, cdfs = 0, []
         for v in probs:
             s += v
             cdfs.append(s / t)
         return vals, cdfs
 
     lineschanged = cdf(desc['lineschanged'])
     fileschanged = cdf(desc['fileschanged'])
     filesadded = cdf(desc['filesadded'])
     dirsadded = cdf(desc['dirsadded'])
     filesremoved = cdf(desc['filesremoved'])
     linelengths = cdf(desc['linelengths'])
     parents = cdf(desc['parents'])
     p1distance = cdf(desc['p1distance'])
     p2distance = cdf(desc['p2distance'])
     interarrival = cdf(desc['interarrival'])
     linesinfilesadded = cdf(desc['linesinfilesadded'])
     tzoffset = cdf(desc['tzoffset'])
 
     dictfile = opts.get('dict') or '/usr/share/dict/words'
     try:
         fp = open(dictfile, 'rU')
     except IOError as err:
         raise error.Abort('%s: %s' % (dictfile, err.strerror))
     words = fp.read().splitlines()
     fp.close()
 
     initdirs = {}
     if desc['initdirs']:
         for k, v in desc['initdirs']:
             initdirs[k.encode('utf-8').replace('.hg', '_hg')] = v
         initdirs = renamedirs(initdirs, words)
     initdirscdf = cdf(initdirs)
 
     def pick(cdf):
         return cdf[0][bisect.bisect_left(cdf[1], random.random())]
 
     def pickpath():
         return os.path.join(pick(initdirscdf), random.choice(words))
 
     def makeline(minimum=0):
         total = max(minimum, pick(linelengths))
         c, l = 0, []
         while c < total:
             w = random.choice(words)
             c += len(w) + 1
             l.append(w)
         return ' '.join(l)
 
     wlock = repo.wlock()
     lock = repo.lock()
 
     nevertouch = set(('.hgsub', '.hgignore', '.hgtags'))
 
     progress = ui.progress
     _synthesizing = _('synthesizing')
     _files = _('initial files')
     _changesets = _('changesets')
 
     # Synthesize a single initial revision adding files to the repo according
     # to the modeled directory structure.
     initcount = int(opts['initfiles'])
     if initcount and initdirs:
         pctx = repo[None].parents()[0]
         dirs = set(pctx.dirs())
         files = {}
 
         def validpath(path):
             # Don't pick filenames which are already directory names.
             if path in dirs:
                 return False
             # Don't pick directories which were used as file names.
             while path:
                 if path in files:
                     return False
                 path = os.path.dirname(path)
             return True
 
         for i in xrange(0, initcount):
             ui.progress(_synthesizing, i, unit=_files, total=initcount)
 
             path = pickpath()
             while not validpath(path):
                 path = pickpath()
             data = '%s contents\n' % path
             files[path] = context.memfilectx(repo, path, data)
             dir = os.path.dirname(path)
             while dir and dir not in dirs:
                 dirs.add(dir)
                 dir = os.path.dirname(dir)
 
         def filectxfn(repo, memctx, path):
             return files[path]
 
         ui.progress(_synthesizing, None)
         message = 'synthesized wide repo with %d files' % (len(files),)
         mc = context.memctx(repo, [pctx.node(), nullid], message,
                             files.iterkeys(), filectxfn, ui.username(),
                             '%d %d' % util.makedate())
         initnode = mc.commit()
         if ui.debugflag:
             hexfn = hex
         else:
             hexfn = short
         ui.status(_('added commit %s with %d files\n')
                   % (hexfn(initnode), len(files)))
 
     # Synthesize incremental revisions to the repository, adding repo depth.
     count = int(opts['count'])
     heads = set(map(repo.changelog.rev, repo.heads()))
     for i in xrange(count):
         progress(_synthesizing, i, unit=_changesets, total=count)
 
         node = repo.changelog.node
         revs = len(repo)
 
         def pickhead(heads, distance):
             if heads:
                 lheads = sorted(heads)
                 rev = revs - min(pick(distance), revs)
                 if rev < lheads[-1]:
                     rev = lheads[bisect.bisect_left(lheads, rev)]
                 else:
                     rev = lheads[-1]
                 return rev, node(rev)
             return nullrev, nullid
 
         r1 = revs - min(pick(p1distance), revs)
         p1 = node(r1)
 
         # the number of heads will grow without bound if we use a pure
         # model, so artificially constrain their proliferation
         toomanyheads = len(heads) > random.randint(1, 20)
         if p2distance[0] and (pick(parents) == 2 or toomanyheads):
             r2, p2 = pickhead(heads.difference([r1]), p2distance)
         else:
             r2, p2 = nullrev, nullid
 
         pl = [p1, p2]
         pctx = repo[r1]
         mf = pctx.manifest()
         mfk = mf.keys()
         changes = {}
         if mfk:
             for __ in xrange(pick(fileschanged)):
                 for __ in xrange(10):
                     fctx = pctx.filectx(random.choice(mfk))
                     path = fctx.path()
                     if not (path in nevertouch or fctx.isbinary() or
                             'l' in fctx.flags()):
                         break
                 lines = fctx.data().splitlines()
                 add, remove = pick(lineschanged)
                 for __ in xrange(remove):
                     if not lines:
                         break
                     del lines[random.randrange(0, len(lines))]
                 for __ in xrange(add):
                     lines.insert(random.randint(0, len(lines)), makeline())
                 path = fctx.path()
                 changes[path] = context.memfilectx(repo, path,
                                                    '\n'.join(lines) + '\n')
             for __ in xrange(pick(filesremoved)):
                 path = random.choice(mfk)
                 for __ in xrange(10):
                     path = random.choice(mfk)
                     if path not in changes:
                         changes[path] = None
                         break
         if filesadded:
             dirs = list(pctx.dirs())
             dirs.insert(0, '')
         for __ in xrange(pick(filesadded)):
             pathstr = ''
             while pathstr in dirs:
                 path = [random.choice(dirs)]
                 if pick(dirsadded):
                     path.append(random.choice(words))
                 path.append(random.choice(words))
                 pathstr = '/'.join(filter(None, path))
             data = '\n'.join(makeline()
                              for __ in xrange(pick(linesinfilesadded))) + '\n'
             changes[pathstr] = context.memfilectx(repo, pathstr, data)
         def filectxfn(repo, memctx, path):
             return changes[path]
         if not changes:
             continue
         if revs:
             date = repo['tip'].date()[0] + pick(interarrival)
         else:
             date = time.time() - (86400 * count)
         # dates in mercurial must be positive, fit in 32-bit signed integers.
         date = min(0x7fffffff, max(0, date))
         user = random.choice(words) + '@' + random.choice(words)
         mc = context.memctx(repo, pl, makeline(minimum=2),
                             sorted(changes.iterkeys()),
                             filectxfn, user, '%d %d' % (date, pick(tzoffset)))
         newnode = mc.commit()
         heads.add(repo.changelog.rev(newnode))
         heads.discard(r1)
         heads.discard(r2)
 
     lock.release()
     wlock.release()
 
 def renamedirs(dirs, words):
     '''Randomly rename the directory names in the per-dir file count dict.'''
     wordgen = itertools.cycle(words)
     replacements = {'': ''}
     def rename(dirpath):
         '''Recursively rename the directory and all path prefixes.
 
         The mapping from path to renamed path is stored for all path prefixes
         as in dynamic programming, ensuring linear runtime and consistent
         renaming regardless of iteration order through the model.
         '''
         if dirpath in replacements:
             return replacements[dirpath]
         head, _ = os.path.split(dirpath)
         if head:
             head = rename(head)
         else:
             head = ''
-        renamed = os.path.join(head, wordgen.next())
+        renamed = os.path.join(head, next(wordgen))
         replacements[dirpath] = renamed
         return renamed
     result = []
     for dirpath, count in dirs.iteritems():
         result.append([rename(dirpath.lstrip(os.sep)), count])
     return result

contrib/undumprevlog

@@ -1,38 +1,46
 #!/usr/bin/env python
 # Undump a dump from dumprevlog
 # $ hg init
 # $ undumprevlog < repo.dump
 
+from __future__ import absolute_import
+
 import sys
-from mercurial import revlog, node, scmutil, util, transaction
+from mercurial import (
+    node,
+    revlog,
+    scmutil,
+    transaction,
+    util,
+)
 
 for fp in (sys.stdin, sys.stdout, sys.stderr):
     util.setbinary(fp)
 
 opener = scmutil.opener('.', False)
 tr = transaction.transaction(sys.stderr.write, opener, {'store': opener},
                              "undump.journal")
 while True:
     l = sys.stdin.readline()
     if not l:
         break
     if l.startswith("file:"):
         f = l[6:-1]
         r = revlog.revlog(opener, f)
         print f
     elif l.startswith("node:"):
         n = node.bin(l[6:-1])
     elif l.startswith("linkrev:"):
         lr = int(l[9:-1])
     elif l.startswith("parents:"):
         p = l[9:-1].split()
         p1 = node.bin(p[0])
         p2 = node.bin(p[1])
     elif l.startswith("length:"):
         length = int(l[8:-1])
         sys.stdin.readline() # start marker
         d = sys.stdin.read(length)
         sys.stdin.readline() # end marker
         r.addrevision(d, tr, lr, p1, p2)
 
 tr.close()

contrib/win32/hgwebdir_wsgi.py

@@ -1,134 +1,135
 # An example WSGI script for IIS/isapi-wsgi to export multiple hgweb repos
 # Copyright 2010-2016 Sune Foldager <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 #
 # Requirements:
 # - Python 2.7, preferably 64 bit
 # - PyWin32 for Python 2.7 (32 or 64 bit)
 # - Mercurial installed from source (python setup.py install) or download the
 #   python module installer from https://www.mercurial-scm.org/wiki/Download
 # - IIS 7 or newer
 #
 #
 # Installation and use:
 #
 # - Download or clone the isapi-wsgi source and run python setup.py install.
 #   https://github.com/hexdump42/isapi-wsgi
 #
 # - Create a directory to hold the shim dll, config files etc. This can reside
 #   inside the standard IIS directory, C:\inetpub, or anywhere else. Copy this
 #   script there.
 #
 # - Run this script (i.e. python hgwebdir_wsgi.py) to get a shim dll. The
 #   shim is identical for all scripts, so you can just copy and rename one
 #   from an earlier run, if you wish. The shim needs to reside in the same
 #   directory as this script.
 #
 # - Start IIS manager and create a new app pool:
 #   .NET CLR Version: No Managed Code
 #   Advanced Settings: Enable 32 Bit Applications, if using 32 bit Python.
 #   You can adjust the identity and maximum worker processes if you wish. This
 #   setup works fine with multiple worker processes.
 #
 # - Create an IIS application where your hgwebdir is to be served from.
 #   Assign it the app pool you just created and point its physical path to the
 #   directory you created.
 #
 # - In the application, remove all handler mappings and setup a wildcard script
 #   handler mapping of type IsapiModule with the shim dll as its executable.
 #   This file MUST reside in the same directory as the shim. The easiest way
 #   to do all this is to close IIS manager, place a web.config file in your
 #   directory and start IIS manager again. The file should contain:
 #
 #   <?xml version="1.0" encoding="UTF-8"?>
 #   <configuration>
 #       <system.webServer>
 #           <handlers accessPolicy="Read, Script">
 #               <clear />
 #               <add name="hgwebdir" path="*" verb="*" modules="IsapiModule"
 #                    scriptProcessor="C:\your\directory\_hgwebdir_wsgi.dll"
 #                    resourceType="Unspecified" requireAccess="None"
 #                    preCondition="bitness64" />
 #           </handlers>
 #       </system.webServer>
 #   </configuration>
 #
 #   Where "bitness64" should be replaced with "bitness32" for 32 bit Python.
 #
 # - Edit ISAPI And CGI Restrictions on the web server (global setting). Add a
 #   restriction pointing to your shim dll and allow it to run.
 #
 # - Create a configuration file in your directory and adjust the configuration
 #   variables below to match your needs. Example configuration:
 #
 #   [web]
 #   style = gitweb
 #   push_ssl = false
 #   allow_push = *
 #   encoding = utf8
 #
 #   [server]
 #   validate = true
 #
 #   [paths]
 #   repo1 = c:\your\directory\repo1
 #   repo2 = c:\your\directory\repo2
 #
 # - Restart the web server and see if things are running.
 #
 
+from __future__ import absolute_import
+
 # Configuration file location
 hgweb_config = r'c:\your\directory\wsgi.config'
 
 # Global settings for IIS path translation
 path_strip = 0   # Strip this many path elements off (when using url rewrite)
 path_prefix = 1  # This many path elements are prefixes (depends on the
                  # virtual path of the IIS application).
 
-from __future__ import absolute_import
 import sys
 
 # Adjust python path if this is not a system-wide install
 #sys.path.insert(0, r'C:\your\custom\hg\build\lib.win32-2.7')
 
 # Enable tracing. Run 'python -m win32traceutil' to debug
 if getattr(sys, 'isapidllhandle', None) is not None:
     import win32traceutil
     win32traceutil.SetupForPrint # silence unused import warning
 
 import isapi_wsgi
 from mercurial.hgweb.hgwebdir_mod import hgwebdir
 
 # Example tweak: Replace isapi_wsgi's handler to provide better error message
 # Other stuff could also be done here, like logging errors etc.
 class WsgiHandler(isapi_wsgi.IsapiWsgiHandler):
     error_status = '500 Internal Server Error' # less silly error message
 
 isapi_wsgi.IsapiWsgiHandler = WsgiHandler
 
 # Only create the hgwebdir instance once
 application = hgwebdir(hgweb_config)
 
 def handler(environ, start_response):
 
     # Translate IIS's weird URLs
     url = environ['SCRIPT_NAME'] + environ['PATH_INFO']
     paths = url[1:].split('/')[path_strip:]
     script_name = '/' + '/'.join(paths[:path_prefix])
     path_info = '/'.join(paths[path_prefix:])
     if path_info:
         path_info = '/' + path_info
     environ['SCRIPT_NAME'] = script_name
     environ['PATH_INFO'] = path_info
 
     return application(environ, start_response)
 
 def __ExtensionFactory__():
     return isapi_wsgi.ISAPISimpleHandler(handler)
 
 if __name__=='__main__':
     from isapi.install import ISAPIParameters, HandleCommandLine
     params = ISAPIParameters()
     HandleCommandLine(params)

contrib/win32/mercurial.ini

@@ -1,98 +1,97
 ; System-wide Mercurial config file.
 ;
 ; !!!  Do Not Edit This File !!!
 ;
 ; This file will be replaced by the installer on every upgrade.
 ; Editing this file can cause strange side effects on Vista.
 ;
 ; http://bitbucket.org/tortoisehg/stable/issue/135
 ;
 ; To change settings you see in this file, override (or enable) them in
 ; your user Mercurial.ini file, where USERNAME is your Windows user name:
 ;
 ; XP or older    - C:\Documents and Settings\USERNAME\Mercurial.ini
 ; Vista or later - C:\Users\USERNAME\Mercurial.ini
 
 
 [ui]
 ; editor used to enter commit logs, etc.  Most text editors will work.
 editor = notepad
 ; show changed files and be a bit more verbose if True
 ; verbose = True
  
 ; username data to appear in commits
 ; it usually takes the form: Joe User <[e-mail]>
 ; username = Joe User <[e-mail]>
 
 ; In order to push/pull over ssh you must specify an ssh tool
 ;ssh = "C:\Progra~1\TortoiseSVN\bin\TortoisePlink.exe" -ssh -2
 ;ssh = C:\cygwin\bin\ssh
 
 ;
 ; For more information about mercurial extensions, start here
 ; http://www.selenic.com/mercurial/wiki/index.cgi/UsingExtensions
 ;
 ; Extensions shipped with Mercurial
 ;
 [extensions]
 ;acl =
 ;blackbox =
 ;bugzilla =
 ;children =
 ;churn =
 ;color =
 ;convert =
 ;eol =
 ;extdiff =
 ;fetch =
 ;gpg =
-;hgcia =
 ;hgk =
 ;highlight = 
 ;histedit =
 ;interhg =
 ;largefiles =
 ;keyword =
 ;mq =
 ;notify =
 ;pager =
 ;patchbomb =
 ;progress =
 ;purge =
 ;rebase =
 ;record =
 ;relink =
 ;schemes =
 ;share =
 ;shelve =
 ;transplant =
 ;win32mbcs =
 ;zeroconf =
 
 ;
 ; Define external diff commands
 ;
 [extdiff]
 ;cmd.bc3diff = C:\Program Files\Beyond Compare 3\BCompare.exe
 ;cmd.vdiff = C:\Progra~1\TortoiseSVN\bin\TortoiseMerge.exe
 ;cmd.vimdiff = gvim.exe
 ;opts.vimdiff = -f "+next" "+execute 'DirDiff' fnameescape(argv(0)) fnameescape(argv(1))"
 
 
 [hgk]
 ; Replace the following with your path to hgk, uncomment it and 
 ; install ActiveTcl (or another win32 port like tclkit)
 ; path="C:\Program Files\Mercurial\Contrib\hgk.tcl"
 ; vdiff=vdiff
  
 
 ;
 ; The git extended diff format can represent binary files, file
 ; permission changes, and rename information that the normal patch format
 ; cannot describe.  However it is also not compatible with tools which
 ; expect normal patches. so enable git patches at your own risk.
 ;
 [diff]
 ;git = false
 ;nodates = false
 

doc/docchecker

@@ -1,63 +1,66
 #!/usr/bin/env python
 #
 # docchecker - look for problematic markup
 #
 # Copyright 2016 timeless <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import, print_function
+
+import re
 import sys
-import re
 
 leadingline = re.compile(r'(^\s*)(\S.*)$')
 
 checks = [
   (r""":hg:`[^`]*'[^`]*`""",
     """warning: please avoid nesting ' in :hg:`...`"""),
   (r'\w:hg:`',
     'warning: please have a space before :hg:'),
   (r"""(?:[^a-z][^'.])hg ([^,;"`]*'(?!hg)){2}""",
     '''warning: please use " instead of ' for hg ... "..."'''),
 ]
 
 def check(line):
     messages = []
     for match, msg in checks:
         if re.search(match, line):
             messages.append(msg)
     if messages:
         print(line)
         for msg in messages:
             print(msg)
 
 def work(file):
     (llead, lline) = ('', '')
 
     for line in file:
         # this section unwraps lines
         match = leadingline.match(line)
         if not match:
             check(lline)
             (llead, lline) = ('', '')
             continue
 
         lead, line = match.group(1), match.group(2)
         if (lead == llead):
             if (lline != ''):
                 lline += ' ' + line
             else:
                 lline = line
         else:
             check(lline)
             (llead, lline) = (lead, line)
     check(lline)
 
 def main():
     for f in sys.argv[1:]:
         try:
             with open(f) as file:
                 work(file)
         except BaseException as e:
             print("failed to process %s: %s" % (f, e))
 
 main()

doc/gendoc.py

@@ -1,224 +1,224
 #!/usr/bin/env python
 """usage: %s DOC ...
 
 where DOC is the name of a document
 """
 
 from __future__ import absolute_import
 
 import os
 import sys
 import textwrap
 
 # This script is executed during installs and may not have C extensions
 # available. Relax C module requirements.
 os.environ['HGMODULEPOLICY'] = 'allow'
 # import from the live mercurial repo
 sys.path.insert(0, "..")
 from mercurial import demandimport; demandimport.enable()
 from mercurial import (
     commands,
     extensions,
     help,
     minirst,
     ui as uimod,
 )
 from mercurial.i18n import (
     gettext,
     _,
 )
 
 table = commands.table
 globalopts = commands.globalopts
 helptable = help.helptable
 loaddoc = help.loaddoc
 
 def get_desc(docstr):
     if not docstr:
         return "", ""
     # sanitize
     docstr = docstr.strip("\n")
     docstr = docstr.rstrip()
     shortdesc = docstr.splitlines()[0].strip()
 
     i = docstr.find("\n")
     if i != -1:
         desc = docstr[i + 2:]
     else:
         desc = shortdesc
 
     desc = textwrap.dedent(desc)
 
     return (shortdesc, desc)
 
 def get_opts(opts):
     for opt in opts:
         if len(opt) == 5:
             shortopt, longopt, default, desc, optlabel = opt
         else:
             shortopt, longopt, default, desc = opt
             optlabel = _("VALUE")
         allopts = []
         if shortopt:
             allopts.append("-%s" % shortopt)
         if longopt:
             allopts.append("--%s" % longopt)
         if isinstance(default, list):
             allopts[-1] += " <%s[+]>" % optlabel
         elif (default is not None) and not isinstance(default, bool):
             allopts[-1] += " <%s>" % optlabel
         if '\n' in desc:
             # only remove line breaks and indentation
             desc = ' '.join(l.lstrip() for l in desc.split('\n'))
         desc += default and _(" (default: %s)") % default or ""
         yield (", ".join(allopts), desc)
 
 def get_cmd(cmd, cmdtable):
     d = {}
     attr = cmdtable[cmd]
     cmds = cmd.lstrip("^").split("|")
 
     d['cmd'] = cmds[0]
     d['aliases'] = cmd.split("|")[1:]
     d['desc'] = get_desc(gettext(attr[0].__doc__))
     d['opts'] = list(get_opts(attr[1]))
 
     s = 'hg ' + cmds[0]
     if len(attr) > 2:
         if not attr[2].startswith('hg'):
             s += ' ' + attr[2]
         else:
             s = attr[2]
     d['synopsis'] = s.strip()
 
     return d
 
 def showdoc(ui):
     # print options
     ui.write(minirst.section(_("Options")))
     multioccur = False
     for optstr, desc in get_opts(globalopts):
         ui.write("%s\n    %s\n\n" % (optstr, desc))
         if optstr.endswith("[+]>"):
             multioccur = True
     if multioccur:
         ui.write(_("\n[+] marked option can be specified multiple times\n"))
         ui.write("\n")
 
     # print cmds
     ui.write(minirst.section(_("Commands")))
     commandprinter(ui, table, minirst.subsection)
 
     # print help topics
     # The config help topic is included in the hgrc.5 man page.
     helpprinter(ui, helptable, minirst.section, exclude=['config'])
 
     ui.write(minirst.section(_("Extensions")))
     ui.write(_("This section contains help for extensions that are "
                "distributed together with Mercurial. Help for other "
                "extensions is available in the help system."))
-    ui.write("\n\n"
+    ui.write(("\n\n"
              ".. contents::\n"
              "   :class: htmlonly\n"
              "   :local:\n"
-             "   :depth: 1\n\n")
+             "   :depth: 1\n\n"))
 
     for extensionname in sorted(allextensionnames()):
         mod = extensions.load(ui, extensionname, None)
         ui.write(minirst.subsection(extensionname))
         ui.write("%s\n\n" % gettext(mod.__doc__))
         cmdtable = getattr(mod, 'cmdtable', None)
         if cmdtable:
             ui.write(minirst.subsubsection(_('Commands')))
             commandprinter(ui, cmdtable, minirst.subsubsubsection)
 
 def showtopic(ui, topic):
     extrahelptable = [
         (["common"], '', loaddoc('common')),
         (["hg.1"], '', loaddoc('hg.1')),
         (["hg-ssh.8"], '', loaddoc('hg-ssh.8')),
         (["hgignore.5"], '', loaddoc('hgignore.5')),
         (["hgrc.5"], '', loaddoc('hgrc.5')),
         (["hgignore.5.gendoc"], '', loaddoc('hgignore')),
         (["hgrc.5.gendoc"], '', loaddoc('config')),
     ]
     helpprinter(ui, helptable + extrahelptable, None, include=[topic])
 
 def helpprinter(ui, helptable, sectionfunc, include=[], exclude=[]):
     for names, sec, doc in helptable:
         if exclude and names[0] in exclude:
             continue
         if include and names[0] not in include:
             continue
         for name in names:
             ui.write(".. _%s:\n" % name)
         ui.write("\n")
         if sectionfunc:
             ui.write(sectionfunc(sec))
         if callable(doc):
             doc = doc(ui)
         ui.write(doc)
         ui.write("\n")
 
 def commandprinter(ui, cmdtable, sectionfunc):
     h = {}
     for c, attr in cmdtable.items():
         f = c.split("|")[0]
         f = f.lstrip("^")
         h[f] = c
     cmds = h.keys()
     cmds.sort()
 
     for f in cmds:
         if f.startswith("debug"):
             continue
         d = get_cmd(h[f], cmdtable)
         ui.write(sectionfunc(d['cmd']))
         # short description
         ui.write(d['desc'][0])
         # synopsis
         ui.write("::\n\n")
         synopsislines = d['synopsis'].splitlines()
         for line in synopsislines:
             # some commands (such as rebase) have a multi-line
             # synopsis
             ui.write("   %s\n" % line)
         ui.write('\n')
         # description
         ui.write("%s\n\n" % d['desc'][1])
         # options
         opt_output = list(d['opts'])
         if opt_output:
             opts_len = max([len(line[0]) for line in opt_output])
             ui.write(_("Options:\n\n"))
             multioccur = False
             for optstr, desc in opt_output:
                 if desc:
                     s = "%-*s  %s" % (opts_len, optstr, desc)
                 else:
                     s = optstr
                 ui.write("%s\n" % s)
                 if optstr.endswith("[+]>"):
                     multioccur = True
             if multioccur:
                 ui.write(_("\n[+] marked option can be specified"
                            " multiple times\n"))
             ui.write("\n")
         # aliases
         if d['aliases']:
             ui.write(_("    aliases: %s\n\n") % " ".join(d['aliases']))
 
 
 def allextensionnames():
     return extensions.enabled().keys() + extensions.disabled().keys()
 
 if __name__ == "__main__":
     doc = 'hg.1.gendoc'
     if len(sys.argv) > 1:
         doc = sys.argv[1]
 
     ui = uimod.ui()
     if doc == 'hg.1.gendoc':
         showdoc(ui)
     else:
         showtopic(ui, sys.argv[1])

doc/hgmanpage.py

@@ -1,1115 +1,1115
 # -*- coding: utf-8 -*-
 # $Id: manpage.py 6110 2009-08-31 14:40:33Z grubert $
 # Author: Engelbert Gruber <[e-mail]>
 # Copyright: This module is put into the public domain.
 
 """
 Simple man page writer for reStructuredText.
 
 Man pages (short for "manual pages") contain system documentation on unix-like
 systems. The pages are grouped in numbered sections:
 
  1 executable programs and shell commands
  2 system calls
  3 library functions
  4 special files
  5 file formats
  6 games
  7 miscellaneous
  8 system administration
 
 Man pages are written in *troff*, a text file formatting system.
 
 See http://www.tldp.org/HOWTO/Man-Page for a start.
 
 Man pages have no subsections only parts.
 Standard parts
 
   NAME ,
   SYNOPSIS ,
   DESCRIPTION ,
   OPTIONS ,
   FILES ,
   SEE ALSO ,
   BUGS ,
 
 and
 
   AUTHOR .
 
 A unix-like system keeps an index of the DESCRIPTIONs, which is accesable
 by the command whatis or apropos.
 
 """
 from __future__ import absolute_import
 
 __docformat__ = 'reStructuredText'
 
 import inspect
 import re
 
 from docutils import (
     languages,
     nodes,
     writers,
 )
 try:
     import roman
 except ImportError:
     from docutils.utils import roman
 import inspect
 
 FIELD_LIST_INDENT = 7
 DEFINITION_LIST_INDENT = 7
 OPTION_LIST_INDENT = 7
 BLOCKQOUTE_INDENT = 3.5
 
 # Define two macros so man/roff can calculate the
 # indent/unindent margins by itself
 MACRO_DEF = (r""".
 .nr rst2man-indent-level 0
 .
 .de1 rstReportMargin
 \\$1 \\n[an-margin]
 level \\n[rst2man-indent-level]
 level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 -
 \\n[rst2man-indent0]
 \\n[rst2man-indent1]
 \\n[rst2man-indent2]
 ..
 .de1 INDENT
 .\" .rstReportMargin pre:
 . RS \\$1
 . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
 . nr rst2man-indent-level +1
 .\" .rstReportMargin post:
 ..
 .de UNINDENT
 . RE
 .\" indent \\n[an-margin]
 .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .nr rst2man-indent-level -1
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
 """)
 
 class Writer(writers.Writer):
 
     supported = ('manpage')
     """Formats this writer supports."""
 
     output = None
     """Final translated form of `document`."""
 
     def __init__(self):
         writers.Writer.__init__(self)
         self.translator_class = Translator
 
     def translate(self):
         visitor = self.translator_class(self.document)
         self.document.walkabout(visitor)
         self.output = visitor.astext()
 
 
 class Table(object):
     def __init__(self):
         self._rows = []
         self._options = ['center']
         self._tab_char = '\t'
         self._coldefs = []
     def new_row(self):
         self._rows.append([])
     def append_separator(self, separator):
         """Append the separator for table head."""
         self._rows.append([separator])
     def append_cell(self, cell_lines):
         """cell_lines is an array of lines"""
         start = 0
         if len(cell_lines) > 0 and cell_lines[0] == '.sp\n':
             start = 1
         self._rows[-1].append(cell_lines[start:])
         if len(self._coldefs) < len(self._rows[-1]):
             self._coldefs.append('l')
     def _minimize_cell(self, cell_lines):
         """Remove leading and trailing blank and ``.sp`` lines"""
         while (cell_lines and cell_lines[0] in ('\n', '.sp\n')):
             del cell_lines[0]
         while (cell_lines and cell_lines[-1] in ('\n', '.sp\n')):
             del cell_lines[-1]
     def as_list(self):
         text = ['.TS\n']
         text.append(' '.join(self._options) + ';\n')
         text.append('|%s|.\n' % ('|'.join(self._coldefs)))
         for row in self._rows:
             # row = array of cells. cell = array of lines.
             text.append('_\n')       # line above
             text.append('T{\n')
             for i in range(len(row)):
                 cell = row[i]
                 self._minimize_cell(cell)
                 text.extend(cell)
                 if not text[-1].endswith('\n'):
                     text[-1] += '\n'
                 if i < len(row) - 1:
                     text.append('T}'+self._tab_char+'T{\n')
                 else:
                     text.append('T}\n')
         text.append('_\n')
         text.append('.TE\n')
         return text
 
 class Translator(nodes.NodeVisitor):
     """"""
 
     words_and_spaces = re.compile(r'\S+| +|\n')
     document_start = """Man page generated from reStructuredText."""
 
     def __init__(self, document):
         nodes.NodeVisitor.__init__(self, document)
         self.settings = settings = document.settings
         lcode = settings.language_code
         arglen = len(inspect.getargspec(languages.get_language)[0])
         if arglen == 2:
             self.language = languages.get_language(lcode,
                                                    self.document.reporter)
         else:
             self.language = languages.get_language(lcode)
         self.head = []
         self.body = []
         self.foot = []
         self.section_level = 0
         self.context = []
         self.topic_class = ''
         self.colspecs = []
         self.compact_p = 1
         self.compact_simple = None
         # the list style "*" bullet or "#" numbered
         self._list_char = []
         # writing the header .TH and .SH NAME is postboned after
         # docinfo.
         self._docinfo = {
                 "title" : "", "title_upper": "",
                 "subtitle" : "",
                 "manual_section" : "", "manual_group" : "",
                 "author" : [],
                 "date" : "",
                 "copyright" : "",
                 "version" : "",
                     }
         self._docinfo_keys = []     # a list to keep the sequence as in source.
         self._docinfo_names = {}    # to get name from text not normalized.
         self._in_docinfo = None
         self._active_table = None
         self._in_literal = False
         self.header_written = 0
         self._line_block = 0
         self.authors = []
         self.section_level = 0
         self._indent = [0]
         # central definition of simple processing rules
         # what to output on : visit, depart
         # Do not use paragraph requests ``.PP`` because these set indentation.
         # use ``.sp``. Remove superfluous ``.sp`` in ``astext``.
         #
         # Fonts are put on a stack, the top one is used.
         # ``.ft P`` or ``\\fP`` pop from stack.
         # ``B`` bold, ``I`` italic, ``R`` roman should be available.
         # Hopefully ``C`` courier too.
         self.defs = {
                 'indent' : ('.INDENT %.1f\n', '.UNINDENT\n'),
                 'definition_list_item' : ('.TP', ''),
                 'field_name' : ('.TP\n.B ', '\n'),
                 'literal' : ('\\fB', '\\fP'),
                 'literal_block' : ('.sp\n.nf\n.ft C\n', '\n.ft P\n.fi\n'),
 
                 'option_list_item' : ('.TP\n', ''),
 
                 'reference' : (r'\%', r'\:'),
                 'emphasis': ('\\fI', '\\fP'),
                 'strong' : ('\\fB', '\\fP'),
                 'term' : ('\n.B ', '\n'),
                 'title_reference' : ('\\fI', '\\fP'),
 
                 'topic-title' : ('.SS ',),
                 'sidebar-title' : ('.SS ',),
 
                 'problematic' : ('\n.nf\n', '\n.fi\n'),
                     }
         # NOTE don't specify the newline before a dot-command, but ensure
         # it is there.
 
     def comment_begin(self, text):
         """Return commented version of the passed text WITHOUT end of
         line/comment."""
         prefix = '.\\" '
         out_text = ''.join(
             [(prefix + in_line + '\n')
             for in_line in text.split('\n')])
         return out_text
 
     def comment(self, text):
         """Return commented version of the passed text."""
         return self.comment_begin(text)+'.\n'
 
     def ensure_eol(self):
         """Ensure the last line in body is terminated by new line."""
         if self.body[-1][-1] != '\n':
             self.body.append('\n')
 
     def astext(self):
         """Return the final formatted document as a string."""
         if not self.header_written:
             # ensure we get a ".TH" as viewers require it.
             self.head.append(self.header())
         # filter body
         for i in xrange(len(self.body) - 1, 0, -1):
             # remove superfluous vertical gaps.
             if self.body[i] == '.sp\n':
                 if self.body[i - 1][:4] in ('.BI ','.IP '):
                     self.body[i] = '.\n'
                 elif (self.body[i - 1][:3] == '.B ' and
                     self.body[i - 2][:4] == '.TP\n'):
                     self.body[i] = '.\n'
                 elif (self.body[i - 1] == '\n' and
                     self.body[i - 2][0] != '.' and
                     (self.body[i - 3][:7] == '.TP\n.B '
                         or self.body[i - 3][:4] == '\n.B ')
                      ):
                     self.body[i] = '.\n'
         return ''.join(self.head + self.body + self.foot)
 
     def deunicode(self, text):
         text = text.replace(u'\xa0', '\\ ')
         text = text.replace(u'\u2020', '\\(dg')
         return text
 
     def visit_Text(self, node):
         text = node.astext()
         text = text.replace('\\','\\e')
         replace_pairs = [
             (u'-', ur'\-'),
             (u'\'', ur'\(aq'),
             (u'´', ur'\''),
             (u'`', ur'\(ga'),
             ]
         for (in_char, out_markup) in replace_pairs:
             text = text.replace(in_char, out_markup)
         # unicode
         text = self.deunicode(text)
         if self._in_literal:
             # prevent interpretation of "." at line start
             if text[0] == '.':
                 text = '\\&' + text
             text = text.replace('\n.', '\n\\&.')
         self.body.append(text)
 
     def depart_Text(self, node):
         pass
 
     def list_start(self, node):
         class enum_char(object):
             enum_style = {
                     'bullet'     : '\\(bu',
                     'emdash'     : '\\(em',
                      }
 
             def __init__(self, style):
                 self._style = style
                 if 'start' in node:
                     self._cnt = node['start'] - 1
                 else:
                     self._cnt = 0
                 self._indent = 2
                 if style == 'arabic':
                     # indentation depends on number of children
                     # and start value.
                     self._indent = len(str(len(node.children)))
                     self._indent += len(str(self._cnt)) + 1
                 elif style == 'loweralpha':
                     self._cnt += ord('a') - 1
                     self._indent = 3
                 elif style == 'upperalpha':
                     self._cnt += ord('A') - 1
                     self._indent = 3
                 elif style.endswith('roman'):
                     self._indent = 5
 
             def next(self):
                 if self._style == 'bullet':
                     return self.enum_style[self._style]
                 elif self._style == 'emdash':
                     return self.enum_style[self._style]
                 self._cnt += 1
                 # TODO add prefix postfix
                 if self._style == 'arabic':
                     return "%d." % self._cnt
                 elif self._style in ('loweralpha', 'upperalpha'):
                     return "%c." % self._cnt
                 elif self._style.endswith('roman'):
                     res = roman.toRoman(self._cnt) + '.'
                     if self._style.startswith('upper'):
                         return res.upper()
                     return res.lower()
                 else:
                     return "%d." % self._cnt
             def get_width(self):
                 return self._indent
             def __repr__(self):
                 return 'enum_style-%s' % list(self._style)
 
         if 'enumtype' in node:
             self._list_char.append(enum_char(node['enumtype']))
         else:
             self._list_char.append(enum_char('bullet'))
         if len(self._list_char) > 1:
             # indent nested lists
             self.indent(self._list_char[-2].get_width())
         else:
             self.indent(self._list_char[-1].get_width())
 
     def list_end(self):
         self.dedent()
         self._list_char.pop()
 
     def header(self):
         tmpl = (".TH %(title_upper)s %(manual_section)s"
                 " \"%(date)s\" \"%(version)s\" \"%(manual_group)s\"\n"
                 ".SH NAME\n"
                 "%(title)s \- %(subtitle)s\n")
         return tmpl % self._docinfo
 
     def append_header(self):
         """append header with .TH and .SH NAME"""
         # NOTE before everything
         # .TH title_upper section date source manual
         if self.header_written:
             return
         self.body.append(self.header())
         self.body.append(MACRO_DEF)
         self.header_written = 1
 
     def visit_address(self, node):
         self.visit_docinfo_item(node, 'address')
 
     def depart_address(self, node):
         pass
 
     def visit_admonition(self, node, name=None):
         if name:
             self.body.append('.IP %s\n' %
                         self.language.labels.get(name, name))
 
     def depart_admonition(self, node):
         self.body.append('.RE\n')
 
     def visit_attention(self, node):
         self.visit_admonition(node, 'attention')
 
     depart_attention = depart_admonition
 
     def visit_docinfo_item(self, node, name):
         if name == 'author':
             self._docinfo[name].append(node.astext())
         else:
             self._docinfo[name] = node.astext()
         self._docinfo_keys.append(name)
-        raise nodes.SkipNode
+        raise nodes.SkipNode()
 
     def depart_docinfo_item(self, node):
         pass
 
     def visit_author(self, node):
         self.visit_docinfo_item(node, 'author')
 
     depart_author = depart_docinfo_item
 
     def visit_authors(self, node):
         # _author is called anyway.
         pass
 
     def depart_authors(self, node):
         pass
 
     def visit_block_quote(self, node):
         # BUG/HACK: indent always uses the _last_ indentation,
         # thus we need two of them.
         self.indent(BLOCKQOUTE_INDENT)
         self.indent(0)
 
     def depart_block_quote(self, node):
         self.dedent()
         self.dedent()
 
     def visit_bullet_list(self, node):
         self.list_start(node)
 
     def depart_bullet_list(self, node):
         self.list_end()
 
     def visit_caption(self, node):
         pass
 
     def depart_caption(self, node):
         pass
 
     def visit_caution(self, node):
         self.visit_admonition(node, 'caution')
 
     depart_caution = depart_admonition
 
     def visit_citation(self, node):
         num, text = node.astext().split(None, 1)
         num = num.strip()
         self.body.append('.IP [%s] 5\n' % num)
 
     def depart_citation(self, node):
         pass
 
     def visit_citation_reference(self, node):
         self.body.append('['+node.astext()+']')
-        raise nodes.SkipNode
+        raise nodes.SkipNode()
 
     def visit_classifier(self, node):
         pass
 
     def depart_classifier(self, node):
         pass
 
     def visit_colspec(self, node):
         self.colspecs.append(node)
 
     def depart_colspec(self, node):
         pass
 
     def write_colspecs(self):
         self.body.append("%s.\n" % ('L '*len(self.colspecs)))
 
     def visit_comment(self, node,
                       sub=re.compile('-(?=-)').sub):
         self.body.append(self.comment(node.astext()))
-        raise nodes.SkipNode
+        raise nodes.SkipNode()
 
     def visit_contact(self, node):
         self.visit_docinfo_item(node, 'contact')
 
     depart_contact = depart_docinfo_item
 
     def visit_container(self, node):
         pass
 
     def depart_container(self, node):
         pass
 
     def visit_compound(self, node):
         pass
 
     def depart_compound(self, node):
         pass
 
     def visit_copyright(self, node):
         self.visit_docinfo_item(node, 'copyright')
 
     def visit_danger(self, node):
         self.visit_admonition(node, 'danger')
 
     depart_danger = depart_admonition
 
     def visit_date(self, node):
         self.visit_docinfo_item(node, 'date')
 
     def visit_decoration(self, node):
         pass
 
     def depart_decoration(self, node):
         pass
 
     def visit_definition(self, node):
         pass
 
     def depart_definition(self, node):
         pass
 
     def visit_definition_list(self, node):
         self.indent(DEFINITION_LIST_INDENT)
 
     def depart_definition_list(self, node):
         self.dedent()
 
     def visit_definition_list_item(self, node):
         self.body.append(self.defs['definition_list_item'][0])
 
     def depart_definition_list_item(self, node):
         self.body.append(self.defs['definition_list_item'][1])
 
     def visit_description(self, node):
         pass
 
     def depart_description(self, node):
         pass
 
     def visit_docinfo(self, node):
         self._in_docinfo = 1
 
     def depart_docinfo(self, node):
         self._in_docinfo = None
         # NOTE nothing should be written before this
         self.append_header()
 
     def visit_doctest_block(self, node):
         self.body.append(self.defs['literal_block'][0])
         self._in_literal = True
 
     def depart_doctest_block(self, node):
         self._in_literal = False
         self.body.append(self.defs['literal_block'][1])
 
     def visit_document(self, node):
         # no blank line between comment and header.
         self.body.append(self.comment(self.document_start).rstrip()+'\n')
         # writing header is postboned
         self.header_written = 0
 
     def depart_document(self, node):
         if self._docinfo['author']:
             self.body.append('.SH AUTHOR\n%s\n'
                     % ', '.join(self._docinfo['author']))
         skip = ('author', 'copyright', 'date',
                 'manual_group', 'manual_section',
                 'subtitle',
                 'title', 'title_upper', 'version')
         for name in self._docinfo_keys:
             if name == 'address':
                 self.body.append("\n%s:\n%s%s.nf\n%s\n.fi\n%s%s" % (
                                     self.language.labels.get(name, name),
                                     self.defs['indent'][0] % 0,
                                     self.defs['indent'][0] % BLOCKQOUTE_INDENT,
                                     self._docinfo[name],
                                     self.defs['indent'][1],
                                     self.defs['indent'][1]))
             elif name not in skip:
                 if name in self._docinfo_names:
                     label = self._docinfo_names[name]
                 else:
                     label = self.language.labels.get(name, name)
                 self.body.append("\n%s: %s\n" % (label, self._docinfo[name]))
         if self._docinfo['copyright']:
             self.body.append('.SH COPYRIGHT\n%s\n'
                     % self._docinfo['copyright'])
         self.body.append(self.comment(
                         'Generated by docutils manpage writer.\n'))
 
     def visit_emphasis(self, node):
         self.body.append(self.defs['emphasis'][0])
 
     def depart_emphasis(self, node):
         self.body.append(self.defs['emphasis'][1])
 
     def visit_entry(self, node):
         # a cell in a table row
         if 'morerows' in node:
             self.document.reporter.warning('"table row spanning" not supported',
                     base_node=node)
         if 'morecols' in node:
             self.document.reporter.warning(
                     '"table cell spanning" not supported', base_node=node)
         self.context.append(len(self.body))
 
     def depart_entry(self, node):
         start = self.context.pop()
         self._active_table.append_cell(self.body[start:])
         del self.body[start:]
 
     def visit_enumerated_list(self, node):
         self.list_start(node)
 
     def depart_enumerated_list(self, node):
         self.list_end()
 
     def visit_error(self, node):
         self.visit_admonition(node, 'error')
 
     depart_error = depart_admonition
 
     def visit_field(self, node):
         pass
 
     def depart_field(self, node):
         pass
 
     def visit_field_body(self, node):
         if self._in_docinfo:
             name_normalized = self._field_name.lower().replace(" ","_")
             self._docinfo_names[name_normalized] = self._field_name
             self.visit_docinfo_item(node, name_normalized)
-            raise nodes.SkipNode
+            raise nodes.SkipNode()
 
     def depart_field_body(self, node):
         pass
 
     def visit_field_list(self, node):
         self.indent(FIELD_LIST_INDENT)
 
     def depart_field_list(self, node):
         self.dedent()
 
     def visit_field_name(self, node):
         if self._in_docinfo:
             self._field_name = node.astext()
-            raise nodes.SkipNode
+            raise nodes.SkipNode()
         else:
             self.body.append(self.defs['field_name'][0])
 
     def depart_field_name(self, node):
         self.body.append(self.defs['field_name'][1])
 
     def visit_figure(self, node):
         self.indent(2.5)
         self.indent(0)
 
     def depart_figure(self, node):
         self.dedent()
         self.dedent()
 
     def visit_footer(self, node):
         self.document.reporter.warning('"footer" not supported',
                 base_node=node)
 
     def depart_footer(self, node):
         pass
 
     def visit_footnote(self, node):
         num, text = node.astext().split(None, 1)
         num = num.strip()
         self.body.append('.IP [%s] 5\n' % self.deunicode(num))
 
     def depart_footnote(self, node):
         pass
 
     def footnote_backrefs(self, node):
         self.document.reporter.warning('"footnote_backrefs" not supported',
                 base_node=node)
 
     def visit_footnote_reference(self, node):
         self.body.append('['+self.deunicode(node.astext())+']')
-        raise nodes.SkipNode
+        raise nodes.SkipNode()
 
     def depart_footnote_reference(self, node):
         pass
 
     def visit_generated(self, node):
         pass
 
     def depart_generated(self, node):
         pass
 
     def visit_header(self, node):
-        raise NotImplementedError, node.astext()
+        raise NotImplementedError(node.astext())
 
     def depart_header(self, node):
         pass
 
     def visit_hint(self, node):
         self.visit_admonition(node, 'hint')
 
     depart_hint = depart_admonition
 
     def visit_subscript(self, node):
         self.body.append('\\s-2\\d')
 
     def depart_subscript(self, node):
         self.body.append('\\u\\s0')
 
     def visit_superscript(self, node):
         self.body.append('\\s-2\\u')
 
     def depart_superscript(self, node):
         self.body.append('\\d\\s0')
 
     def visit_attribution(self, node):
         self.body.append('\\(em ')
 
     def depart_attribution(self, node):
         self.body.append('\n')
 
     def visit_image(self, node):
         self.document.reporter.warning('"image" not supported',
                 base_node=node)
         text = []
         if 'alt' in node.attributes:
             text.append(node.attributes['alt'])
         if 'uri' in node.attributes:
             text.append(node.attributes['uri'])
         self.body.append('[image: %s]\n' % ('/'.join(text)))
-        raise nodes.SkipNode
+        raise nodes.SkipNode()
 
     def visit_important(self, node):
         self.visit_admonition(node, 'important')
 
     depart_important = depart_admonition
 
     def visit_label(self, node):
         # footnote and citation
         if (isinstance(node.parent, nodes.footnote)
             or isinstance(node.parent, nodes.citation)):
-            raise nodes.SkipNode
+            raise nodes.SkipNode()
         self.document.reporter.warning('"unsupported "label"',
                 base_node=node)
         self.body.append('[')
 
     def depart_label(self, node):
         self.body.append(']\n')
 
     def visit_legend(self, node):
         pass
 
     def depart_legend(self, node):
         pass
 
     # WHAT should we use .INDENT, .UNINDENT ?
     def visit_line_block(self, node):
         self._line_block += 1
         if self._line_block == 1:
             self.body.append('.sp\n')
             self.body.append('.nf\n')
         else:
             self.body.append('.in +2\n')
 
     def depart_line_block(self, node):
         self._line_block -= 1
         if self._line_block == 0:
             self.body.append('.fi\n')
             self.body.append('.sp\n')
         else:
             self.body.append('.in -2\n')
 
     def visit_line(self, node):
         pass
 
     def depart_line(self, node):
         self.body.append('\n')
 
     def visit_list_item(self, node):
         # man 7 man argues to use ".IP" instead of ".TP"
         self.body.append('.IP %s %d\n' % (
-                self._list_char[-1].next(),
+                next(self._list_char[-1]),
                 self._list_char[-1].get_width(),))
 
     def depart_list_item(self, node):
         pass
 
     def visit_literal(self, node):
         self.body.append(self.defs['literal'][0])
 
     def depart_literal(self, node):
         self.body.append(self.defs['literal'][1])
 
     def visit_literal_block(self, node):
         self.body.append(self.defs['literal_block'][0])
         self._in_literal = True
 
     def depart_literal_block(self, node):
         self._in_literal = False
         self.body.append(self.defs['literal_block'][1])
 
     def visit_meta(self, node):
-        raise NotImplementedError, node.astext()
+        raise NotImplementedError(node.astext())
 
     def depart_meta(self, node):
         pass
 
     def visit_note(self, node):
         self.visit_admonition(node, 'note')
 
     depart_note = depart_admonition
 
     def indent(self, by=0.5):
         # if we are in a section ".SH" there already is a .RS
         step = self._indent[-1]
         self._indent.append(by)
         self.body.append(self.defs['indent'][0] % step)
 
     def dedent(self):
         self._indent.pop()
         self.body.append(self.defs['indent'][1])
 
     def visit_option_list(self, node):
         self.indent(OPTION_LIST_INDENT)
 
     def depart_option_list(self, node):
         self.dedent()
 
     def visit_option_list_item(self, node):
         # one item of the list
         self.body.append(self.defs['option_list_item'][0])
 
     def depart_option_list_item(self, node):
         self.body.append(self.defs['option_list_item'][1])
 
     def visit_option_group(self, node):
         # as one option could have several forms it is a group
         # options without parameter bold only, .B, -v
         # options with parameter bold italic, .BI, -f file
         #
         # we do not know if .B or .BI
         self.context.append('.B')           # blind guess
         self.context.append(len(self.body)) # to be able to insert later
         self.context.append(0)              # option counter
 
     def depart_option_group(self, node):
         self.context.pop()  # the counter
         start_position = self.context.pop()
         text = self.body[start_position:]
         del self.body[start_position:]
         self.body.append('%s%s\n' % (self.context.pop(), ''.join(text)))
 
     def visit_option(self, node):
         # each form of the option will be presented separately
         if self.context[-1] > 0:
             self.body.append(', ')
         if self.context[-3] == '.BI':
             self.body.append('\\')
         self.body.append(' ')
 
     def depart_option(self, node):
         self.context[-1] += 1
 
     def visit_option_string(self, node):
         # do not know if .B or .BI
         pass
 
     def depart_option_string(self, node):
         pass
 
     def visit_option_argument(self, node):
         self.context[-3] = '.BI' # bold/italic alternate
         if node['delimiter'] != ' ':
             self.body.append('\\fB%s ' % node['delimiter'])
         elif self.body[len(self.body) - 1].endswith('='):
             # a blank only means no blank in output, just changing font
             self.body.append(' ')
         else:
             # blank backslash blank, switch font then a blank
             self.body.append(' \\ ')
 
     def depart_option_argument(self, node):
         pass
 
     def visit_organization(self, node):
         self.visit_docinfo_item(node, 'organization')
 
     def depart_organization(self, node):
         pass
 
     def visit_paragraph(self, node):
         # ``.PP`` : Start standard indented paragraph.
         # ``.LP`` : Start block paragraph, all except the first.
         # ``.P [type]``  : Start paragraph type.
         # NOTE don't use paragraph starts because they reset indentation.
         # ``.sp`` is only vertical space
         self.ensure_eol()
         self.body.append('.sp\n')
 
     def depart_paragraph(self, node):
         self.body.append('\n')
 
     def visit_problematic(self, node):
         self.body.append(self.defs['problematic'][0])
 
     def depart_problematic(self, node):
         self.body.append(self.defs['problematic'][1])
 
     def visit_raw(self, node):
         if node.get('format') == 'manpage':
             self.body.append(node.astext() + "\n")
         # Keep non-manpage raw text out of output:
-        raise nodes.SkipNode
+        raise nodes.SkipNode()
 
     def visit_reference(self, node):
         """E.g. link or email address."""
         self.body.append(self.defs['reference'][0])
 
     def depart_reference(self, node):
         self.body.append(self.defs['reference'][1])
 
     def visit_revision(self, node):
         self.visit_docinfo_item(node, 'revision')
 
     depart_revision = depart_docinfo_item
 
     def visit_row(self, node):
         self._active_table.new_row()
 
     def depart_row(self, node):
         pass
 
     def visit_section(self, node):
         self.section_level += 1
 
     def depart_section(self, node):
         self.section_level -= 1
 
     def visit_status(self, node):
         self.visit_docinfo_item(node, 'status')
 
     depart_status = depart_docinfo_item
 
     def visit_strong(self, node):
         self.body.append(self.defs['strong'][0])
 
     def depart_strong(self, node):
         self.body.append(self.defs['strong'][1])
 
     def visit_substitution_definition(self, node):
         """Internal only."""
-        raise nodes.SkipNode
+        raise nodes.SkipNode()
 
     def visit_substitution_reference(self, node):
         self.document.reporter.warning('"substitution_reference" not supported',
                 base_node=node)
 
     def visit_subtitle(self, node):
         if isinstance(node.parent, nodes.sidebar):
             self.body.append(self.defs['strong'][0])
         elif isinstance(node.parent, nodes.document):
             self.visit_docinfo_item(node, 'subtitle')
         elif isinstance(node.parent, nodes.section):
             self.body.append(self.defs['strong'][0])
 
     def depart_subtitle(self, node):
         # document subtitle calls SkipNode
         self.body.append(self.defs['strong'][1]+'\n.PP\n')
 
     def visit_system_message(self, node):
         # TODO add report_level
         #if node['level'] < self.document.reporter['writer'].report_level:
         #    Level is too low to display:
         #    raise nodes.SkipNode
         attr = {}
         if node.hasattr('id'):
             attr['name'] = node['id']
         if node.hasattr('line'):
             line = ', line %s' % node['line']
         else:
             line = ''
         self.body.append('.IP "System Message: %s/%s (%s:%s)"\n'
                          % (node['type'], node['level'], node['source'], line))
 
     def depart_system_message(self, node):
         pass
 
     def visit_table(self, node):
         self._active_table = Table()
 
     def depart_table(self, node):
         self.ensure_eol()
         self.body.extend(self._active_table.as_list())
         self._active_table = None
 
     def visit_target(self, node):
         # targets are in-document hyper targets, without any use for man-pages.
-        raise nodes.SkipNode
+        raise nodes.SkipNode()
 
     def visit_tbody(self, node):
         pass
 
     def depart_tbody(self, node):
         pass
 
     def visit_term(self, node):
         self.body.append(self.defs['term'][0])
 
     def depart_term(self, node):
         self.body.append(self.defs['term'][1])
 
     def visit_tgroup(self, node):
         pass
 
     def depart_tgroup(self, node):
         pass
 
     def visit_thead(self, node):
         # MAYBE double line '='
         pass
 
     def depart_thead(self, node):
         # MAYBE double line '='
         pass
 
     def visit_tip(self, node):
         self.visit_admonition(node, 'tip')
 
     depart_tip = depart_admonition
 
     def visit_title(self, node):
         if isinstance(node.parent, nodes.topic):
             self.body.append(self.defs['topic-title'][0])
         elif isinstance(node.parent, nodes.sidebar):
             self.body.append(self.defs['sidebar-title'][0])
         elif isinstance(node.parent, nodes.admonition):
             self.body.append('.IP "')
         elif self.section_level == 0:
             self._docinfo['title'] = node.astext()
             # document title for .TH
             self._docinfo['title_upper'] = node.astext().upper()
-            raise nodes.SkipNode
+            raise nodes.SkipNode()
         elif self.section_level == 1:
             self.body.append('.SH ')
             for n in node.traverse(nodes.Text):
                 n.parent.replace(n, nodes.Text(n.astext().upper()))
         else:
             self.body.append('.SS ')
 
     def depart_title(self, node):
         if isinstance(node.parent, nodes.admonition):
             self.body.append('"')
         self.body.append('\n')
 
     def visit_title_reference(self, node):
         """inline citation reference"""
         self.body.append(self.defs['title_reference'][0])
 
     def depart_title_reference(self, node):
         self.body.append(self.defs['title_reference'][1])
 
     def visit_topic(self, node):
         pass
 
     def depart_topic(self, node):
         pass
 
     def visit_sidebar(self, node):
         pass
 
     def depart_sidebar(self, node):
         pass
 
     def visit_rubric(self, node):
         pass
 
     def depart_rubric(self, node):
         pass
 
     def visit_transition(self, node):
         # .PP      Begin a new paragraph and reset prevailing indent.
         # .sp N    leaves N lines of blank space.
         # .ce      centers the next line
         self.body.append('\n.sp\n.ce\n----\n')
 
     def depart_transition(self, node):
         self.body.append('\n.ce 0\n.sp\n')
 
     def visit_version(self, node):
         self.visit_docinfo_item(node, 'version')
 
     def visit_warning(self, node):
         self.visit_admonition(node, 'warning')
 
     depart_warning = depart_admonition
 
     def unimplemented_visit(self, node):
         raise NotImplementedError('visiting unimplemented node type: %s'
                                   % node.__class__.__name__)
 
 # vim: set fileencoding=utf-8 et ts=4 ai :

hg

@@ -1,43 +1,45
 #!/usr/bin/env python
 #
 # mercurial - scalable distributed SCM
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import os
 import sys
 
 if os.environ.get('HGUNICODEPEDANTRY', False):
-    reload(sys)
-    sys.setdefaultencoding("undefined")
-
+    try:
+        reload(sys)
+        sys.setdefaultencoding("undefined")
+    except NameError:
+        pass
 
 libdir = '@LIBDIR@'
 
 if libdir != '@' 'LIBDIR' '@':
     if not os.path.isabs(libdir):
         libdir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                               libdir)
         libdir = os.path.abspath(libdir)
     sys.path.insert(0, libdir)
 
 # enable importing on demand to reduce startup time
 try:
-    from mercurial import demandimport; demandimport.enable()
+    if sys.version_info[0] < 3:
+        from mercurial import demandimport; demandimport.enable()
 except ImportError:
-    import sys
     sys.stderr.write("abort: couldn't find mercurial libraries in [%s]\n" %
                      ' '.join(sys.path))
     sys.stderr.write("(check your install and PYTHONPATH)\n")
     sys.exit(-1)
 
 import mercurial.util
 import mercurial.dispatch
 
 for fp in (sys.stdin, sys.stdout, sys.stderr):
     mercurial.util.setbinary(fp)
 
 mercurial.dispatch.run()

hgext/automv.py

@@ -1,100 +1,100
 # automv.py
 #
 # Copyright 2013-2016 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """Check for unrecorded moves at commit time (EXPERIMENTAL)
 
 This extension checks at commit/amend time if any of the committed files
 comes from an unrecorded mv.
 
 The threshold at which a file is considered a move can be set with the
 ``automv.similarity`` config option. This option takes a percentage between 0
 (disabled) and 100 (files must be identical), the default is 95.
 
 """
 
 # Using 95 as a default similarity is based on an analysis of the mercurial
 # repositories of the cpython, mozilla-central & mercurial repositories, as
 # well as 2 very large facebook repositories. At 95 50% of all potential
 # missed moves would be caught, as well as correspond with 87% of all
 # explicitly marked moves.  Together, 80% of moved files are 95% similar or
 # more.
 #
 # See http://markmail.org/thread/5pxnljesvufvom57 for context.
 
 from __future__ import absolute_import
 
+from mercurial.i18n import _
 from mercurial import (
     commands,
     copies,
     error,
     extensions,
     scmutil,
     similar
 )
-from mercurial.i18n import _
 
 def extsetup(ui):
     entry = extensions.wrapcommand(
         commands.table, 'commit', mvcheck)
     entry[1].append(
         ('', 'no-automv', None,
          _('disable automatic file move detection')))
 
 def mvcheck(orig, ui, repo, *pats, **opts):
     """Hook to check for moves at commit time"""
     renames = None
     disabled = opts.pop('no_automv', False)
     if not disabled:
         threshold = ui.configint('automv', 'similarity', 95)
         if not 0 <= threshold <= 100:
             raise error.Abort(_('automv.similarity must be between 0 and 100'))
         if threshold > 0:
             match = scmutil.match(repo[None], pats, opts)
             added, removed = _interestingfiles(repo, match)
             renames = _findrenames(repo, match, added, removed,
                                    threshold / 100.0)
 
     with repo.wlock():
         if renames is not None:
             scmutil._markchanges(repo, (), (), renames)
         return orig(ui, repo, *pats, **opts)
 
 def _interestingfiles(repo, matcher):
     """Find what files were added or removed in this commit.
 
     Returns a tuple of two lists: (added, removed). Only files not *already*
     marked as moved are included in the added list.
 
     """
     stat = repo.status(match=matcher)
     added = stat[1]
     removed = stat[2]
 
     copy = copies._forwardcopies(repo['.'], repo[None], matcher)
     # remove the copy files for which we already have copy info
     added = [f for f in added if f not in copy]
 
     return added, removed
 
 def _findrenames(repo, matcher, added, removed, similarity):
     """Find what files in added are really moved files.
 
     Any file named in removed that is at least similarity% similar to a file
     in added is seen as a rename.
 
     """
     renames = {}
     if similarity > 0:
         for src, dst, score in similar.findrenames(
                 repo, added, removed, similarity):
             if repo.ui.verbose:
                 repo.ui.status(
                     _('detected move of %s as %s (%d%% similar)\n') % (
                         matcher.rel(src), matcher.rel(dst), score * 100))
             renames[dst] = src
     if renames:
         repo.ui.status(_('detected move of %d files\n') % len(renames))
     return renames

hgext/bugzilla.py

@@ -1,927 +1,928
 # bugzilla.py - bugzilla integration for mercurial
 #
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 # Copyright 2011-4 Jim Hague <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''hooks for integrating with the Bugzilla bug tracker
 
 This hook extension adds comments on bugs in Bugzilla when changesets
 that refer to bugs by Bugzilla ID are seen. The comment is formatted using
 the Mercurial template mechanism.
 
 The bug references can optionally include an update for Bugzilla of the
 hours spent working on the bug. Bugs can also be marked fixed.
 
 Three basic modes of access to Bugzilla are provided:
 
 1. Access via the Bugzilla XMLRPC interface. Requires Bugzilla 3.4 or later.
 
 2. Check data via the Bugzilla XMLRPC interface and submit bug change
    via email to Bugzilla email interface. Requires Bugzilla 3.4 or later.
 
 3. Writing directly to the Bugzilla database. Only Bugzilla installations
    using MySQL are supported. Requires Python MySQLdb.
 
 Writing directly to the database is susceptible to schema changes, and
 relies on a Bugzilla contrib script to send out bug change
 notification emails. This script runs as the user running Mercurial,
 must be run on the host with the Bugzilla install, and requires
 permission to read Bugzilla configuration details and the necessary
 MySQL user and password to have full access rights to the Bugzilla
 database. For these reasons this access mode is now considered
 deprecated, and will not be updated for new Bugzilla versions going
 forward. Only adding comments is supported in this access mode.
 
 Access via XMLRPC needs a Bugzilla username and password to be specified
 in the configuration. Comments are added under that username. Since the
 configuration must be readable by all Mercurial users, it is recommended
 that the rights of that user are restricted in Bugzilla to the minimum
 necessary to add comments. Marking bugs fixed requires Bugzilla 4.0 and later.
 
 Access via XMLRPC/email uses XMLRPC to query Bugzilla, but sends
 email to the Bugzilla email interface to submit comments to bugs.
 The From: address in the email is set to the email address of the Mercurial
 user, so the comment appears to come from the Mercurial user. In the event
 that the Mercurial user email is not recognized by Bugzilla as a Bugzilla
 user, the email associated with the Bugzilla username used to log into
 Bugzilla is used instead as the source of the comment. Marking bugs fixed
 works on all supported Bugzilla versions.
 
 Configuration items common to all access modes:
 
 bugzilla.version
   The access type to use. Values recognized are:
 
   :``xmlrpc``:       Bugzilla XMLRPC interface.
   :``xmlrpc+email``: Bugzilla XMLRPC and email interfaces.
   :``3.0``:          MySQL access, Bugzilla 3.0 and later.
   :``2.18``:         MySQL access, Bugzilla 2.18 and up to but not
                      including 3.0.
   :``2.16``:         MySQL access, Bugzilla 2.16 and up to but not
                      including 2.18.
 
 bugzilla.regexp
   Regular expression to match bug IDs for update in changeset commit message.
   It must contain one "()" named group ``<ids>`` containing the bug
   IDs separated by non-digit characters. It may also contain
   a named group ``<hours>`` with a floating-point number giving the
   hours worked on the bug. If no named groups are present, the first
   "()" group is assumed to contain the bug IDs, and work time is not
   updated. The default expression matches ``Bug 1234``, ``Bug no. 1234``,
   ``Bug number 1234``, ``Bugs 1234,5678``, ``Bug 1234 and 5678`` and
   variations thereof, followed by an hours number prefixed by ``h`` or
   ``hours``, e.g. ``hours 1.5``. Matching is case insensitive.
 
 bugzilla.fixregexp
   Regular expression to match bug IDs for marking fixed in changeset
   commit message. This must contain a "()" named group ``<ids>` containing
   the bug IDs separated by non-digit characters. It may also contain
   a named group ``<hours>`` with a floating-point number giving the
   hours worked on the bug. If no named groups are present, the first
   "()" group is assumed to contain the bug IDs, and work time is not
   updated. The default expression matches ``Fixes 1234``, ``Fixes bug 1234``,
   ``Fixes bugs 1234,5678``, ``Fixes 1234 and 5678`` and
   variations thereof, followed by an hours number prefixed by ``h`` or
   ``hours``, e.g. ``hours 1.5``. Matching is case insensitive.
 
 bugzilla.fixstatus
   The status to set a bug to when marking fixed. Default ``RESOLVED``.
 
 bugzilla.fixresolution
   The resolution to set a bug to when marking fixed. Default ``FIXED``.
 
 bugzilla.style
   The style file to use when formatting comments.
 
 bugzilla.template
   Template to use when formatting comments. Overrides style if
   specified. In addition to the usual Mercurial keywords, the
   extension specifies:
 
   :``{bug}``:     The Bugzilla bug ID.
   :``{root}``:    The full pathname of the Mercurial repository.
   :``{webroot}``: Stripped pathname of the Mercurial repository.
   :``{hgweb}``:   Base URL for browsing Mercurial repositories.
 
   Default ``changeset {node|short} in repo {root} refers to bug
   {bug}.\\ndetails:\\n\\t{desc|tabindent}``
 
 bugzilla.strip
   The number of path separator characters to strip from the front of
   the Mercurial repository path (``{root}`` in templates) to produce
   ``{webroot}``. For example, a repository with ``{root}``
   ``/var/local/my-project`` with a strip of 2 gives a value for
   ``{webroot}`` of ``my-project``. Default 0.
 
 web.baseurl
   Base URL for browsing Mercurial repositories. Referenced from
   templates as ``{hgweb}``.
 
 Configuration items common to XMLRPC+email and MySQL access modes:
 
 bugzilla.usermap
   Path of file containing Mercurial committer email to Bugzilla user email
   mappings. If specified, the file should contain one mapping per
   line::
 
     committer = Bugzilla user
 
   See also the ``[usermap]`` section.
 
 The ``[usermap]`` section is used to specify mappings of Mercurial
 committer email to Bugzilla user email. See also ``bugzilla.usermap``.
 Contains entries of the form ``committer = Bugzilla user``.
 
 XMLRPC access mode configuration:
 
 bugzilla.bzurl
   The base URL for the Bugzilla installation.
   Default ``http://localhost/bugzilla``.
 
 bugzilla.user
   The username to use to log into Bugzilla via XMLRPC. Default
   ``bugs``.
 
 bugzilla.password
   The password for Bugzilla login.
 
 XMLRPC+email access mode uses the XMLRPC access mode configuration items,
 and also:
 
 bugzilla.bzemail
   The Bugzilla email address.
 
 In addition, the Mercurial email settings must be configured. See the
 documentation in hgrc(5), sections ``[email]`` and ``[smtp]``.
 
 MySQL access mode configuration:
 
 bugzilla.host
   Hostname of the MySQL server holding the Bugzilla database.
   Default ``localhost``.
 
 bugzilla.db
   Name of the Bugzilla database in MySQL. Default ``bugs``.
 
 bugzilla.user
   Username to use to access MySQL server. Default ``bugs``.
 
 bugzilla.password
   Password to use to access MySQL server.
 
 bugzilla.timeout
   Database connection timeout (seconds). Default 5.
 
 bugzilla.bzuser
   Fallback Bugzilla user name to record comments with, if changeset
   committer cannot be found as a Bugzilla user.
 
 bugzilla.bzdir
    Bugzilla install directory. Used by default notify. Default
    ``/var/www/html/bugzilla``.
 
 bugzilla.notify
   The command to run to get Bugzilla to send bug change notification
   emails. Substitutes from a map with 3 keys, ``bzdir``, ``id`` (bug
   id) and ``user`` (committer bugzilla email). Default depends on
   version; from 2.18 it is "cd %(bzdir)s && perl -T
   contrib/sendbugmail.pl %(id)s %(user)s".
 
 Activating the extension::
 
     [extensions]
     bugzilla =
 
     [hooks]
     # run bugzilla hook on every change pulled or pushed in here
     incoming.bugzilla = python:hgext.bugzilla.hook
 
 Example configurations:
 
 XMLRPC example configuration. This uses the Bugzilla at
 ``http://my-project.org/bugzilla``, logging in as user
 ``[e-mail]`` with password ``plugh``. It is used with a
 collection of Mercurial repositories in ``/var/local/hg/repos/``,
 with a web interface at ``http://my-project.org/hg``. ::
 
     [bugzilla]
     bzurl=http://my-project.org/bugzilla
     user=[e-mail]
     password=plugh
     version=xmlrpc
     template=Changeset {node|short} in {root|basename}.
              {hgweb}/{webroot}/rev/{node|short}\\n
              {desc}\\n
     strip=5
 
     [web]
     baseurl=http://my-project.org/hg
 
 XMLRPC+email example configuration. This uses the Bugzilla at
 ``http://my-project.org/bugzilla``, logging in as user
 ``[e-mail]`` with password ``plugh``. It is used with a
 collection of Mercurial repositories in ``/var/local/hg/repos/``,
 with a web interface at ``http://my-project.org/hg``. Bug comments
 are sent to the Bugzilla email address
 ``[e-mail]``. ::
 
     [bugzilla]
     bzurl=http://my-project.org/bugzilla
     user=[e-mail]
     password=plugh
     version=xmlrpc+email
     bzemail=[e-mail]
     template=Changeset {node|short} in {root|basename}.
              {hgweb}/{webroot}/rev/{node|short}\\n
              {desc}\\n
     strip=5
 
     [web]
     baseurl=http://my-project.org/hg
 
     [usermap]
     [e-mail]=[e-mail]
 
 MySQL example configuration. This has a local Bugzilla 3.2 installation
 in ``/opt/bugzilla-3.2``. The MySQL database is on ``localhost``,
 the Bugzilla database name is ``bugs`` and MySQL is
 accessed with MySQL username ``bugs`` password ``XYZZY``. It is used
 with a collection of Mercurial repositories in ``/var/local/hg/repos/``,
 with a web interface at ``http://my-project.org/hg``. ::
 
     [bugzilla]
     host=localhost
     password=XYZZY
     version=3.0
     bzuser=[e-mail]
     bzdir=/opt/bugzilla-3.2
     template=Changeset {node|short} in {root|basename}.
              {hgweb}/{webroot}/rev/{node|short}\\n
              {desc}\\n
     strip=5
 
     [web]
     baseurl=http://my-project.org/hg
 
     [usermap]
     [e-mail]=[e-mail]
 
 All the above add a comment to the Bugzilla bug record of the form::
 
     Changeset 3b16791d6642 in repository-name.
     http://my-project.org/hg/repository-name/rev/3b16791d6642
 
     Changeset commit comment. Bug 1234.
 '''
 
 from __future__ import absolute_import
 
 import re
 import time
-import urlparse
-import xmlrpclib
 
 from mercurial.i18n import _
 from mercurial.node import short
 from mercurial import (
     cmdutil,
     error,
     mail,
     util,
 )
 
+urlparse = util.urlparse
+xmlrpclib = util.xmlrpclib
+
 # Note for extension authors: ONLY specify testedwith = 'internal' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'internal'
 
 class bzaccess(object):
     '''Base class for access to Bugzilla.'''
 
     def __init__(self, ui):
         self.ui = ui
         usermap = self.ui.config('bugzilla', 'usermap')
         if usermap:
             self.ui.readconfig(usermap, sections=['usermap'])
 
     def map_committer(self, user):
         '''map name of committer to Bugzilla user name.'''
         for committer, bzuser in self.ui.configitems('usermap'):
             if committer.lower() == user.lower():
                 return bzuser
         return user
 
     # Methods to be implemented by access classes.
     #
     # 'bugs' is a dict keyed on bug id, where values are a dict holding
     # updates to bug state. Recognized dict keys are:
     #
     # 'hours': Value, float containing work hours to be updated.
     # 'fix':   If key present, bug is to be marked fixed. Value ignored.
 
     def filter_real_bug_ids(self, bugs):
         '''remove bug IDs that do not exist in Bugzilla from bugs.'''
         pass
 
     def filter_cset_known_bug_ids(self, node, bugs):
         '''remove bug IDs where node occurs in comment text from bugs.'''
         pass
 
     def updatebug(self, bugid, newstate, text, committer):
         '''update the specified bug. Add comment text and set new states.
 
         If possible add the comment as being from the committer of
         the changeset. Otherwise use the default Bugzilla user.
         '''
         pass
 
     def notify(self, bugs, committer):
         '''Force sending of Bugzilla notification emails.
 
         Only required if the access method does not trigger notification
         emails automatically.
         '''
         pass
 
 # Bugzilla via direct access to MySQL database.
 class bzmysql(bzaccess):
     '''Support for direct MySQL access to Bugzilla.
 
     The earliest Bugzilla version this is tested with is version 2.16.
 
     If your Bugzilla is version 3.4 or above, you are strongly
     recommended to use the XMLRPC access method instead.
     '''
 
     @staticmethod
     def sql_buglist(ids):
         '''return SQL-friendly list of bug ids'''
         return '(' + ','.join(map(str, ids)) + ')'
 
     _MySQLdb = None
 
     def __init__(self, ui):
         try:
             import MySQLdb as mysql
             bzmysql._MySQLdb = mysql
         except ImportError as err:
             raise error.Abort(_('python mysql support not available: %s') % err)
 
         bzaccess.__init__(self, ui)
 
         host = self.ui.config('bugzilla', 'host', 'localhost')
         user = self.ui.config('bugzilla', 'user', 'bugs')
         passwd = self.ui.config('bugzilla', 'password')
         db = self.ui.config('bugzilla', 'db', 'bugs')
         timeout = int(self.ui.config('bugzilla', 'timeout', 5))
         self.ui.note(_('connecting to %s:%s as %s, password %s\n') %
                      (host, db, user, '*' * len(passwd)))
         self.conn = bzmysql._MySQLdb.connect(host=host,
                                                    user=user, passwd=passwd,
                                                    db=db,
                                                    connect_timeout=timeout)
         self.cursor = self.conn.cursor()
         self.longdesc_id = self.get_longdesc_id()
         self.user_ids = {}
         self.default_notify = "cd %(bzdir)s && ./processmail %(id)s %(user)s"
 
     def run(self, *args, **kwargs):
         '''run a query.'''
         self.ui.note(_('query: %s %s\n') % (args, kwargs))
         try:
             self.cursor.execute(*args, **kwargs)
         except bzmysql._MySQLdb.MySQLError:
             self.ui.note(_('failed query: %s %s\n') % (args, kwargs))
             raise
 
     def get_longdesc_id(self):
         '''get identity of longdesc field'''
         self.run('select fieldid from fielddefs where name = "longdesc"')
         ids = self.cursor.fetchall()
         if len(ids) != 1:
             raise error.Abort(_('unknown database schema'))
         return ids[0][0]
 
     def filter_real_bug_ids(self, bugs):
         '''filter not-existing bugs from set.'''
         self.run('select bug_id from bugs where bug_id in %s' %
                  bzmysql.sql_buglist(bugs.keys()))
         existing = [id for (id,) in self.cursor.fetchall()]
         for id in bugs.keys():
             if id not in existing:
                 self.ui.status(_('bug %d does not exist\n') % id)
                 del bugs[id]
 
     def filter_cset_known_bug_ids(self, node, bugs):
         '''filter bug ids that already refer to this changeset from set.'''
         self.run('''select bug_id from longdescs where
                     bug_id in %s and thetext like "%%%s%%"''' %
                  (bzmysql.sql_buglist(bugs.keys()), short(node)))
         for (id,) in self.cursor.fetchall():
             self.ui.status(_('bug %d already knows about changeset %s\n') %
                            (id, short(node)))
             del bugs[id]
 
     def notify(self, bugs, committer):
         '''tell bugzilla to send mail.'''
         self.ui.status(_('telling bugzilla to send mail:\n'))
         (user, userid) = self.get_bugzilla_user(committer)
         for id in bugs.keys():
             self.ui.status(_('  bug %s\n') % id)
             cmdfmt = self.ui.config('bugzilla', 'notify', self.default_notify)
             bzdir = self.ui.config('bugzilla', 'bzdir',
                                    '/var/www/html/bugzilla')
             try:
                 # Backwards-compatible with old notify string, which
                 # took one string. This will throw with a new format
                 # string.
                 cmd = cmdfmt % id
             except TypeError:
                 cmd = cmdfmt % {'bzdir': bzdir, 'id': id, 'user': user}
             self.ui.note(_('running notify command %s\n') % cmd)
             fp = util.popen('(%s) 2>&1' % cmd)
             out = fp.read()
             ret = fp.close()
             if ret:
                 self.ui.warn(out)
                 raise error.Abort(_('bugzilla notify command %s') %
                                  util.explainexit(ret)[0])
         self.ui.status(_('done\n'))
 
     def get_user_id(self, user):
         '''look up numeric bugzilla user id.'''
         try:
             return self.user_ids[user]
         except KeyError:
             try:
                 userid = int(user)
             except ValueError:
                 self.ui.note(_('looking up user %s\n') % user)
                 self.run('''select userid from profiles
                             where login_name like %s''', user)
                 all = self.cursor.fetchall()
                 if len(all) != 1:
                     raise KeyError(user)
                 userid = int(all[0][0])
             self.user_ids[user] = userid
             return userid
 
     def get_bugzilla_user(self, committer):
         '''See if committer is a registered bugzilla user. Return
         bugzilla username and userid if so. If not, return default
         bugzilla username and userid.'''
         user = self.map_committer(committer)
         try:
             userid = self.get_user_id(user)
         except KeyError:
             try:
                 defaultuser = self.ui.config('bugzilla', 'bzuser')
                 if not defaultuser:
                     raise error.Abort(_('cannot find bugzilla user id for %s') %
                                      user)
                 userid = self.get_user_id(defaultuser)
                 user = defaultuser
             except KeyError:
                 raise error.Abort(_('cannot find bugzilla user id for %s or %s')
                                  % (user, defaultuser))
         return (user, userid)
 
     def updatebug(self, bugid, newstate, text, committer):
         '''update bug state with comment text.
 
         Try adding comment as committer of changeset, otherwise as
         default bugzilla user.'''
         if len(newstate) > 0:
             self.ui.warn(_("Bugzilla/MySQL cannot update bug state\n"))
 
         (user, userid) = self.get_bugzilla_user(committer)
         now = time.strftime('%Y-%m-%d %H:%M:%S')
         self.run('''insert into longdescs
                     (bug_id, who, bug_when, thetext)
                     values (%s, %s, %s, %s)''',
                  (bugid, userid, now, text))
         self.run('''insert into bugs_activity (bug_id, who, bug_when, fieldid)
                     values (%s, %s, %s, %s)''',
                  (bugid, userid, now, self.longdesc_id))
         self.conn.commit()
 
 class bzmysql_2_18(bzmysql):
     '''support for bugzilla 2.18 series.'''
 
     def __init__(self, ui):
         bzmysql.__init__(self, ui)
         self.default_notify = \
             "cd %(bzdir)s && perl -T contrib/sendbugmail.pl %(id)s %(user)s"
 
 class bzmysql_3_0(bzmysql_2_18):
     '''support for bugzilla 3.0 series.'''
 
     def __init__(self, ui):
         bzmysql_2_18.__init__(self, ui)
 
     def get_longdesc_id(self):
         '''get identity of longdesc field'''
         self.run('select id from fielddefs where name = "longdesc"')
         ids = self.cursor.fetchall()
         if len(ids) != 1:
             raise error.Abort(_('unknown database schema'))
         return ids[0][0]
 
 # Bugzilla via XMLRPC interface.
 
 class cookietransportrequest(object):
     """A Transport request method that retains cookies over its lifetime.
 
     The regular xmlrpclib transports ignore cookies. Which causes
     a bit of a problem when you need a cookie-based login, as with
     the Bugzilla XMLRPC interface prior to 4.4.3.
 
     So this is a helper for defining a Transport which looks for
     cookies being set in responses and saves them to add to all future
     requests.
     """
 
     # Inspiration drawn from
     # http://blog.godson.in/2010/09/how-to-make-python-xmlrpclib-client.html
     # http://www.itkovian.net/base/transport-class-for-pythons-xml-rpc-lib/
 
     cookies = []
     def send_cookies(self, connection):
         if self.cookies:
             for cookie in self.cookies:
                 connection.putheader("Cookie", cookie)
 
     def request(self, host, handler, request_body, verbose=0):
         self.verbose = verbose
         self.accept_gzip_encoding = False
 
         # issue XML-RPC request
         h = self.make_connection(host)
         if verbose:
             h.set_debuglevel(1)
 
         self.send_request(h, handler, request_body)
         self.send_host(h, host)
         self.send_cookies(h)
         self.send_user_agent(h)
         self.send_content(h, request_body)
 
         # Deal with differences between Python 2.4-2.6 and 2.7.
         # In the former h is a HTTP(S). In the latter it's a
         # HTTP(S)Connection. Luckily, the 2.4-2.6 implementation of
         # HTTP(S) has an underlying HTTP(S)Connection, so extract
         # that and use it.
         try:
             response = h.getresponse()
         except AttributeError:
             response = h._conn.getresponse()
 
         # Add any cookie definitions to our list.
         for header in response.msg.getallmatchingheaders("Set-Cookie"):
             val = header.split(": ", 1)[1]
             cookie = val.split(";", 1)[0]
             self.cookies.append(cookie)
 
         if response.status != 200:
             raise xmlrpclib.ProtocolError(host + handler, response.status,
                                           response.reason, response.msg.headers)
 
         payload = response.read()
         parser, unmarshaller = self.getparser()
         parser.feed(payload)
         parser.close()
 
         return unmarshaller.close()
 
 # The explicit calls to the underlying xmlrpclib __init__() methods are
 # necessary. The xmlrpclib.Transport classes are old-style classes, and
 # it turns out their __init__() doesn't get called when doing multiple
 # inheritance with a new-style class.
 class cookietransport(cookietransportrequest, xmlrpclib.Transport):
     def __init__(self, use_datetime=0):
         if util.safehasattr(xmlrpclib.Transport, "__init__"):
             xmlrpclib.Transport.__init__(self, use_datetime)
 
 class cookiesafetransport(cookietransportrequest, xmlrpclib.SafeTransport):
     def __init__(self, use_datetime=0):
         if util.safehasattr(xmlrpclib.Transport, "__init__"):
             xmlrpclib.SafeTransport.__init__(self, use_datetime)
 
 class bzxmlrpc(bzaccess):
     """Support for access to Bugzilla via the Bugzilla XMLRPC API.
 
     Requires a minimum Bugzilla version 3.4.
     """
 
     def __init__(self, ui):
         bzaccess.__init__(self, ui)
 
         bzweb = self.ui.config('bugzilla', 'bzurl',
                                'http://localhost/bugzilla/')
         bzweb = bzweb.rstrip("/") + "/xmlrpc.cgi"
 
         user = self.ui.config('bugzilla', 'user', 'bugs')
         passwd = self.ui.config('bugzilla', 'password')
 
         self.fixstatus = self.ui.config('bugzilla', 'fixstatus', 'RESOLVED')
         self.fixresolution = self.ui.config('bugzilla', 'fixresolution',
                                             'FIXED')
 
         self.bzproxy = xmlrpclib.ServerProxy(bzweb, self.transport(bzweb))
         ver = self.bzproxy.Bugzilla.version()['version'].split('.')
         self.bzvermajor = int(ver[0])
         self.bzverminor = int(ver[1])
         login = self.bzproxy.User.login({'login': user, 'password': passwd,
                                          'restrict_login': True})
         self.bztoken = login.get('token', '')
 
     def transport(self, uri):
         if urlparse.urlparse(uri, "http")[0] == "https":
             return cookiesafetransport()
         else:
             return cookietransport()
 
     def get_bug_comments(self, id):
         """Return a string with all comment text for a bug."""
         c = self.bzproxy.Bug.comments({'ids': [id],
                                        'include_fields': ['text'],
                                        'token': self.bztoken})
         return ''.join([t['text'] for t in c['bugs'][str(id)]['comments']])
 
     def filter_real_bug_ids(self, bugs):
         probe = self.bzproxy.Bug.get({'ids': sorted(bugs.keys()),
                                       'include_fields': [],
                                       'permissive': True,
                                       'token': self.bztoken,
                                       })
         for badbug in probe['faults']:
             id = badbug['id']
             self.ui.status(_('bug %d does not exist\n') % id)
             del bugs[id]
 
     def filter_cset_known_bug_ids(self, node, bugs):
         for id in sorted(bugs.keys()):
             if self.get_bug_comments(id).find(short(node)) != -1:
                 self.ui.status(_('bug %d already knows about changeset %s\n') %
                                (id, short(node)))
                 del bugs[id]
 
     def updatebug(self, bugid, newstate, text, committer):
         args = {}
         if 'hours' in newstate:
             args['work_time'] = newstate['hours']
 
         if self.bzvermajor >= 4:
             args['ids'] = [bugid]
             args['comment'] = {'body' : text}
             if 'fix' in newstate:
                 args['status'] = self.fixstatus
                 args['resolution'] = self.fixresolution
             args['token'] = self.bztoken
             self.bzproxy.Bug.update(args)
         else:
             if 'fix' in newstate:
                 self.ui.warn(_("Bugzilla/XMLRPC needs Bugzilla 4.0 or later "
                                "to mark bugs fixed\n"))
             args['id'] = bugid
             args['comment'] = text
             self.bzproxy.Bug.add_comment(args)
 
 class bzxmlrpcemail(bzxmlrpc):
     """Read data from Bugzilla via XMLRPC, send updates via email.
 
     Advantages of sending updates via email:
       1. Comments can be added as any user, not just logged in user.
       2. Bug statuses or other fields not accessible via XMLRPC can
          potentially be updated.
 
     There is no XMLRPC function to change bug status before Bugzilla
     4.0, so bugs cannot be marked fixed via XMLRPC before Bugzilla 4.0.
     But bugs can be marked fixed via email from 3.4 onwards.
     """
 
     # The email interface changes subtly between 3.4 and 3.6. In 3.4,
     # in-email fields are specified as '@<fieldname> = <value>'. In
     # 3.6 this becomes '@<fieldname> <value>'. And fieldname @bug_id
     # in 3.4 becomes @id in 3.6. 3.6 and 4.0 both maintain backwards
     # compatibility, but rather than rely on this use the new format for
     # 4.0 onwards.
 
     def __init__(self, ui):
         bzxmlrpc.__init__(self, ui)
 
         self.bzemail = self.ui.config('bugzilla', 'bzemail')
         if not self.bzemail:
             raise error.Abort(_("configuration 'bzemail' missing"))
         mail.validateconfig(self.ui)
 
     def makecommandline(self, fieldname, value):
         if self.bzvermajor >= 4:
             return "@%s %s" % (fieldname, str(value))
         else:
             if fieldname == "id":
                 fieldname = "bug_id"
             return "@%s = %s" % (fieldname, str(value))
 
     def send_bug_modify_email(self, bugid, commands, comment, committer):
         '''send modification message to Bugzilla bug via email.
 
         The message format is documented in the Bugzilla email_in.pl
         specification. commands is a list of command lines, comment is the
         comment text.
 
         To stop users from crafting commit comments with
         Bugzilla commands, specify the bug ID via the message body, rather
         than the subject line, and leave a blank line after it.
         '''
         user = self.map_committer(committer)
         matches = self.bzproxy.User.get({'match': [user],
                                          'token': self.bztoken})
         if not matches['users']:
             user = self.ui.config('bugzilla', 'user', 'bugs')
             matches = self.bzproxy.User.get({'match': [user],
                                              'token': self.bztoken})
             if not matches['users']:
                 raise error.Abort(_("default bugzilla user %s email not found")
                                   % user)
         user = matches['users'][0]['email']
         commands.append(self.makecommandline("id", bugid))
 
         text = "\n".join(commands) + "\n\n" + comment
 
         _charsets = mail._charsets(self.ui)
         user = mail.addressencode(self.ui, user, _charsets)
         bzemail = mail.addressencode(self.ui, self.bzemail, _charsets)
         msg = mail.mimeencode(self.ui, text, _charsets)
         msg['From'] = user
         msg['To'] = bzemail
         msg['Subject'] = mail.headencode(self.ui, "Bug modification", _charsets)
         sendmail = mail.connect(self.ui)
         sendmail(user, bzemail, msg.as_string())
 
     def updatebug(self, bugid, newstate, text, committer):
         cmds = []
         if 'hours' in newstate:
             cmds.append(self.makecommandline("work_time", newstate['hours']))
         if 'fix' in newstate:
             cmds.append(self.makecommandline("bug_status", self.fixstatus))
             cmds.append(self.makecommandline("resolution", self.fixresolution))
         self.send_bug_modify_email(bugid, cmds, text, committer)
 
 class bugzilla(object):
     # supported versions of bugzilla. different versions have
     # different schemas.
     _versions = {
         '2.16': bzmysql,
         '2.18': bzmysql_2_18,
         '3.0':  bzmysql_3_0,
         'xmlrpc': bzxmlrpc,
         'xmlrpc+email': bzxmlrpcemail
         }
 
     _default_bug_re = (r'bugs?\s*,?\s*(?:#|nos?\.?|num(?:ber)?s?)?\s*'
                        r'(?P<ids>(?:\d+\s*(?:,?\s*(?:and)?)?\s*)+)'
                        r'\.?\s*(?:h(?:ours?)?\s*(?P<hours>\d*(?:\.\d+)?))?')
 
     _default_fix_re = (r'fix(?:es)?\s*(?:bugs?\s*)?,?\s*'
                        r'(?:nos?\.?|num(?:ber)?s?)?\s*'
                        r'(?P<ids>(?:#?\d+\s*(?:,?\s*(?:and)?)?\s*)+)'
                        r'\.?\s*(?:h(?:ours?)?\s*(?P<hours>\d*(?:\.\d+)?))?')
 
     def __init__(self, ui, repo):
         self.ui = ui
         self.repo = repo
 
         bzversion = self.ui.config('bugzilla', 'version')
         try:
             bzclass = bugzilla._versions[bzversion]
         except KeyError:
             raise error.Abort(_('bugzilla version %s not supported') %
                              bzversion)
         self.bzdriver = bzclass(self.ui)
 
         self.bug_re = re.compile(
             self.ui.config('bugzilla', 'regexp',
                            bugzilla._default_bug_re), re.IGNORECASE)
         self.fix_re = re.compile(
             self.ui.config('bugzilla', 'fixregexp',
                            bugzilla._default_fix_re), re.IGNORECASE)
         self.split_re = re.compile(r'\D+')
 
     def find_bugs(self, ctx):
         '''return bugs dictionary created from commit comment.
 
         Extract bug info from changeset comments. Filter out any that are
         not known to Bugzilla, and any that already have a reference to
         the given changeset in their comments.
         '''
         start = 0
         hours = 0.0
         bugs = {}
         bugmatch = self.bug_re.search(ctx.description(), start)
         fixmatch = self.fix_re.search(ctx.description(), start)
         while True:
             bugattribs = {}
             if not bugmatch and not fixmatch:
                 break
             if not bugmatch:
                 m = fixmatch
             elif not fixmatch:
                 m = bugmatch
             else:
                 if bugmatch.start() < fixmatch.start():
                     m = bugmatch
                 else:
                     m = fixmatch
             start = m.end()
             if m is bugmatch:
                 bugmatch = self.bug_re.search(ctx.description(), start)
                 if 'fix' in bugattribs:
                     del bugattribs['fix']
             else:
                 fixmatch = self.fix_re.search(ctx.description(), start)
                 bugattribs['fix'] = None
 
             try:
                 ids = m.group('ids')
             except IndexError:
                 ids = m.group(1)
             try:
                 hours = float(m.group('hours'))
                 bugattribs['hours'] = hours
             except IndexError:
                 pass
             except TypeError:
                 pass
             except ValueError:
                 self.ui.status(_("%s: invalid hours\n") % m.group('hours'))
 
             for id in self.split_re.split(ids):
                 if not id:
                     continue
                 bugs[int(id)] = bugattribs
         if bugs:
             self.bzdriver.filter_real_bug_ids(bugs)
         if bugs:
             self.bzdriver.filter_cset_known_bug_ids(ctx.node(), bugs)
         return bugs
 
     def update(self, bugid, newstate, ctx):
         '''update bugzilla bug with reference to changeset.'''
 
         def webroot(root):
             '''strip leading prefix of repo root and turn into
             url-safe path.'''
             count = int(self.ui.config('bugzilla', 'strip', 0))
             root = util.pconvert(root)
             while count > 0:
                 c = root.find('/')
                 if c == -1:
                     break
                 root = root[c + 1:]
                 count -= 1
             return root
 
         mapfile = None
         tmpl = self.ui.config('bugzilla', 'template')
         if not tmpl:
             mapfile = self.ui.config('bugzilla', 'style')
         if not mapfile and not tmpl:
             tmpl = _('changeset {node|short} in repo {root} refers '
                      'to bug {bug}.\ndetails:\n\t{desc|tabindent}')
         t = cmdutil.changeset_templater(self.ui, self.repo,
                                         False, None, tmpl, mapfile, False)
         self.ui.pushbuffer()
         t.show(ctx, changes=ctx.changeset(),
                bug=str(bugid),
                hgweb=self.ui.config('web', 'baseurl'),
                root=self.repo.root,
                webroot=webroot(self.repo.root))
         data = self.ui.popbuffer()
         self.bzdriver.updatebug(bugid, newstate, data, util.email(ctx.user()))
 
     def notify(self, bugs, committer):
         '''ensure Bugzilla users are notified of bug change.'''
         self.bzdriver.notify(bugs, committer)
 
 def hook(ui, repo, hooktype, node=None, **kwargs):
     '''add comment to bugzilla for each changeset that refers to a
     bugzilla bug id. only add a comment once per bug, so same change
     seen multiple times does not fill bug with duplicate data.'''
     if node is None:
         raise error.Abort(_('hook type %s does not pass a changeset id') %
                          hooktype)
     try:
         bz = bugzilla(ui, repo)
         ctx = repo[node]
         bugs = bz.find_bugs(ctx)
         if bugs:
             for bug in bugs:
                 bz.update(bug, bugs[bug], ctx)
             bz.notify(bugs, util.email(ctx.user()))
     except Exception as e:
         raise error.Abort(_('Bugzilla error: %s') % e)

hgext/chgserver.py

@@ -1,712 +1,643
 # chgserver.py - command server extension for cHg
 #
 # Copyright 2011 Yuya Nishihara <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """command server extension for cHg (EXPERIMENTAL)
 
 'S' channel (read/write)
     propagate ui.system() request to client
 
 'attachio' command
     attach client's stdio passed by sendmsg()
 
 'chdir' command
     change current directory
 
 'getpager' command
     checks if pager is enabled and which pager should be executed
 
 'setenv' command
     replace os.environ completely
 
 'setumask' command
     set umask
 
 'validate' command
     reload the config and check if the server is up to date
 
 Config
 ------
 
 ::
 
   [chgserver]
   idletimeout = 3600 # seconds, after which an idle server will exit
   skiphash = False   # whether to skip config or env change checks
 """
 
 from __future__ import absolute_import
 
-import SocketServer
 import errno
-import gc
+import hashlib
 import inspect
 import os
-import random
 import re
+import signal
 import struct
 import sys
-import threading
 import time
-import traceback
 
 from mercurial.i18n import _
 
 from mercurial import (
     cmdutil,
     commands,
     commandserver,
     dispatch,
     error,
     extensions,
     osutil,
     util,
 )
 
 # Note for extension authors: ONLY specify testedwith = 'internal' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'internal'
 
 _log = commandserver.log
 
 def _hashlist(items):
     """return sha1 hexdigest for a list"""
-    return util.sha1(str(items)).hexdigest()
+    return hashlib.sha1(str(items)).hexdigest()
 
 # sensitive config sections affecting confighash
 _configsections = [
+    'alias',  # affects global state commands.table
     'extdiff',  # uisetup will register new commands
     'extensions',
 ]
 
 # sensitive environment variables affecting confighash
 _envre = re.compile(r'''\A(?:
                     CHGHG
                     |HG.*
                     |LANG(?:UAGE)?
                     |LC_.*
                     |LD_.*
                     |PATH
                     |PYTHON.*
                     |TERM(?:INFO)?
                     |TZ
                     )\Z''', re.X)
 
 def _confighash(ui):
     """return a quick hash for detecting config/env changes
 
     confighash is the hash of sensitive config items and environment variables.
 
     for chgserver, it is designed that once confighash changes, the server is
     not qualified to serve its client and should redirect the client to a new
     server. different from mtimehash, confighash change will not mark the
     server outdated and exit since the user can have different configs at the
     same time.
     """
     sectionitems = []
     for section in _configsections:
         sectionitems.append(ui.configitems(section))
     sectionhash = _hashlist(sectionitems)
     envitems = [(k, v) for k, v in os.environ.iteritems() if _envre.match(k)]
     envhash = _hashlist(sorted(envitems))
     return sectionhash[:6] + envhash[:6]
 
 def _getmtimepaths(ui):
     """get a list of paths that should be checked to detect change
 
     The list will include:
     - extensions (will not cover all files for complex extensions)
     - mercurial/__version__.py
     - python binary
     """
     modules = [m for n, m in extensions.extensions(ui)]
     try:
         from mercurial import __version__
         modules.append(__version__)
     except ImportError:
         pass
     files = [sys.executable]
     for m in modules:
         try:
             files.append(inspect.getabsfile(m))
         except TypeError:
             pass
     return sorted(set(files))
 
 def _mtimehash(paths):
     """return a quick hash for detecting file changes
 
     mtimehash calls stat on given paths and calculate a hash based on size and
     mtime of each file. mtimehash does not read file content because reading is
     expensive. therefore it's not 100% reliable for detecting content changes.
     it's possible to return different hashes for same file contents.
     it's also possible to return a same hash for different file contents for
     some carefully crafted situation.
 
     for chgserver, it is designed that once mtimehash changes, the server is
     considered outdated immediately and should no longer provide service.
+
+    mtimehash is not included in confighash because we only know the paths of
+    extensions after importing them (there is imp.find_module but that faces
+    race conditions). We need to calculate confighash without importing.
     """
     def trystat(path):
         try:
             st = os.stat(path)
             return (st.st_mtime, st.st_size)
         except OSError:
             # could be ENOENT, EPERM etc. not fatal in any case
             pass
     return _hashlist(map(trystat, paths))[:12]
 
 class hashstate(object):
     """a structure storing confighash, mtimehash, paths used for mtimehash"""
     def __init__(self, confighash, mtimehash, mtimepaths):
         self.confighash = confighash
         self.mtimehash = mtimehash
         self.mtimepaths = mtimepaths
 
     @staticmethod
     def fromui(ui, mtimepaths=None):
         if mtimepaths is None:
             mtimepaths = _getmtimepaths(ui)
         confighash = _confighash(ui)
         mtimehash = _mtimehash(mtimepaths)
         _log('confighash = %s mtimehash = %s\n' % (confighash, mtimehash))
         return hashstate(confighash, mtimehash, mtimepaths)
 
 # copied from hgext/pager.py:uisetup()
 def _setuppagercmd(ui, options, cmd):
     if not ui.formatted():
         return
 
     p = ui.config("pager", "pager", os.environ.get("PAGER"))
     usepager = False
     always = util.parsebool(options['pager'])
     auto = options['pager'] == 'auto'
 
     if not p:
         pass
     elif always:
         usepager = True
     elif not auto:
         usepager = False
     else:
         attended = ['annotate', 'cat', 'diff', 'export', 'glog', 'log', 'qdiff']
         attend = ui.configlist('pager', 'attend', attended)
         ignore = ui.configlist('pager', 'ignore')
         cmds, _ = cmdutil.findcmd(cmd, commands.table)
 
         for cmd in cmds:
             var = 'attend-%s' % cmd
             if ui.config('pager', var):
                 usepager = ui.configbool('pager', var)
                 break
             if (cmd in attend or
                 (cmd not in ignore and not attend)):
                 usepager = True
                 break
 
     if usepager:
         ui.setconfig('ui', 'formatted', ui.formatted(), 'pager')
         ui.setconfig('ui', 'interactive', False, 'pager')
         return p
 
-_envvarre = re.compile(r'\$[a-zA-Z_]+')
-
-def _clearenvaliases(cmdtable):
-    """Remove stale command aliases referencing env vars; variable expansion
-    is done at dispatch.addaliases()"""
-    for name, tab in cmdtable.items():
-        cmddef = tab[0]
-        if (isinstance(cmddef, dispatch.cmdalias) and
-            not cmddef.definition.startswith('!') and  # shell alias
-            _envvarre.search(cmddef.definition)):
-            del cmdtable[name]
-
 def _newchgui(srcui, csystem):
     class chgui(srcui.__class__):
         def __init__(self, src=None):
             super(chgui, self).__init__(src)
             if src:
                 self._csystem = getattr(src, '_csystem', csystem)
             else:
                 self._csystem = csystem
 
         def system(self, cmd, environ=None, cwd=None, onerr=None,
                    errprefix=None):
             # fallback to the original system method if the output needs to be
             # captured (to self._buffers), or the output stream is not stdout
             # (e.g. stderr, cStringIO), because the chg client is not aware of
             # these situations and will behave differently (write to stdout).
             if (any(s[1] for s in self._bufferstates)
                 or not util.safehasattr(self.fout, 'fileno')
                 or self.fout.fileno() != sys.stdout.fileno()):
                 return super(chgui, self).system(cmd, environ, cwd, onerr,
                                                  errprefix)
             # copied from mercurial/util.py:system()
             self.flush()
             def py2shell(val):
                 if val is None or val is False:
                     return '0'
                 if val is True:
                     return '1'
                 return str(val)
             env = os.environ.copy()
             if environ:
                 env.update((k, py2shell(v)) for k, v in environ.iteritems())
             env['HG'] = util.hgexecutable()
             rc = self._csystem(cmd, env, cwd)
             if rc and onerr:
                 errmsg = '%s %s' % (os.path.basename(cmd.split(None, 1)[0]),
                                     util.explainexit(rc)[0])
                 if errprefix:
                     errmsg = '%s: %s' % (errprefix, errmsg)
                 raise onerr(errmsg)
             return rc
 
     return chgui(srcui)
 
 def _loadnewui(srcui, args):
     newui = srcui.__class__()
     for a in ['fin', 'fout', 'ferr', 'environ']:
         setattr(newui, a, getattr(srcui, a))
     if util.safehasattr(srcui, '_csystem'):
         newui._csystem = srcui._csystem
 
     # internal config: extensions.chgserver
     newui.setconfig('extensions', 'chgserver',
                     srcui.config('extensions', 'chgserver'), '--config')
 
     # command line args
     args = args[:]
     dispatch._parseconfig(newui, dispatch._earlygetopt(['--config'], args))
 
     # stolen from tortoisehg.util.copydynamicconfig()
     for section, name, value in srcui.walkconfig():
         source = srcui.configsource(section, name)
         if ':' in source or source == '--config':
             # path:line or command line
             continue
         if source == 'none':
             # ui.configsource returns 'none' by default
             source = ''
         newui.setconfig(section, name, value, source)
 
     # load wd and repo config, copied from dispatch.py
     cwds = dispatch._earlygetopt(['--cwd'], args)
     cwd = cwds and os.path.realpath(cwds[-1]) or None
     rpath = dispatch._earlygetopt(["-R", "--repository", "--repo"], args)
     path, newlui = dispatch._getlocal(newui, rpath, wd=cwd)
 
     return (newui, newlui)
 
 class channeledsystem(object):
     """Propagate ui.system() request in the following format:
 
     payload length (unsigned int),
     cmd, '\0',
     cwd, '\0',
     envkey, '=', val, '\0',
     ...
     envkey, '=', val
 
     and waits:
 
     exitcode length (unsigned int),
     exitcode (int)
     """
     def __init__(self, in_, out, channel):
         self.in_ = in_
         self.out = out
         self.channel = channel
 
     def __call__(self, cmd, environ, cwd):
         args = [util.quotecommand(cmd), os.path.abspath(cwd or '.')]
         args.extend('%s=%s' % (k, v) for k, v in environ.iteritems())
         data = '\0'.join(args)
         self.out.write(struct.pack('>cI', self.channel, len(data)))
         self.out.write(data)
         self.out.flush()
 
         length = self.in_.read(4)
         length, = struct.unpack('>I', length)
         if length != 4:
             raise error.Abort(_('invalid response'))
         rc, = struct.unpack('>i', self.in_.read(4))
         return rc
 
 _iochannels = [
     # server.ch, ui.fp, mode
     ('cin', 'fin', 'rb'),
     ('cout', 'fout', 'wb'),
     ('cerr', 'ferr', 'wb'),
 ]
 
 class chgcmdserver(commandserver.server):
     def __init__(self, ui, repo, fin, fout, sock, hashstate, baseaddress):
         super(chgcmdserver, self).__init__(
             _newchgui(ui, channeledsystem(fin, fout, 'S')), repo, fin, fout)
         self.clientsock = sock
         self._oldios = []  # original (self.ch, ui.fp, fd) before "attachio"
         self.hashstate = hashstate
         self.baseaddress = baseaddress
         if hashstate is not None:
             self.capabilities = self.capabilities.copy()
             self.capabilities['validate'] = chgcmdserver.validate
 
     def cleanup(self):
+        super(chgcmdserver, self).cleanup()
         # dispatch._runcatch() does not flush outputs if exception is not
         # handled by dispatch._dispatch()
         self.ui.flush()
         self._restoreio()
 
     def attachio(self):
         """Attach to client's stdio passed via unix domain socket; all
         channels except cresult will no longer be used
         """
         # tell client to sendmsg() with 1-byte payload, which makes it
         # distinctive from "attachio\n" command consumed by client.read()
         self.clientsock.sendall(struct.pack('>cI', 'I', 1))
         clientfds = osutil.recvfds(self.clientsock.fileno())
         _log('received fds: %r\n' % clientfds)
 
         ui = self.ui
         ui.flush()
         first = self._saveio()
         for fd, (cn, fn, mode) in zip(clientfds, _iochannels):
             assert fd > 0
             fp = getattr(ui, fn)
             os.dup2(fd, fp.fileno())
             os.close(fd)
             if not first:
                 continue
             # reset buffering mode when client is first attached. as we want
             # to see output immediately on pager, the mode stays unchanged
             # when client re-attached. ferr is unchanged because it should
             # be unbuffered no matter if it is a tty or not.
             if fn == 'ferr':
                 newfp = fp
             else:
                 # make it line buffered explicitly because the default is
                 # decided on first write(), where fout could be a pager.
                 if fp.isatty():
                     bufsize = 1  # line buffered
                 else:
                     bufsize = -1  # system default
                 newfp = os.fdopen(fp.fileno(), mode, bufsize)
                 setattr(ui, fn, newfp)
             setattr(self, cn, newfp)
 
         self.cresult.write(struct.pack('>i', len(clientfds)))
 
     def _saveio(self):
         if self._oldios:
             return False
         ui = self.ui
         for cn, fn, _mode in _iochannels:
             ch = getattr(self, cn)
             fp = getattr(ui, fn)
             fd = os.dup(fp.fileno())
             self._oldios.append((ch, fp, fd))
         return True
 
     def _restoreio(self):
         ui = self.ui
         for (ch, fp, fd), (cn, fn, _mode) in zip(self._oldios, _iochannels):
             newfp = getattr(ui, fn)
             # close newfp while it's associated with client; otherwise it
             # would be closed when newfp is deleted
             if newfp is not fp:
                 newfp.close()
             # restore original fd: fp is open again
             os.dup2(fd, fp.fileno())
             os.close(fd)
             setattr(self, cn, ch)
             setattr(ui, fn, fp)
         del self._oldios[:]
 
     def validate(self):
         """Reload the config and check if the server is up to date
 
         Read a list of '\0' separated arguments.
         Write a non-empty list of '\0' separated instruction strings or '\0'
         if the list is empty.
         An instruction string could be either:
             - "unlink $path", the client should unlink the path to stop the
               outdated server.
             - "redirect $path", the client should attempt to connect to $path
               first. If it does not work, start a new server. It implies
               "reconnect".
             - "exit $n", the client should exit directly with code n.
               This may happen if we cannot parse the config.
             - "reconnect", the client should close the connection and
               reconnect.
         If neither "reconnect" nor "redirect" is included in the instruction
         list, the client can continue with this server after completing all
         the instructions.
         """
         args = self._readlist()
         try:
             self.ui, lui = _loadnewui(self.ui, args)
         except error.ParseError as inst:
             dispatch._formatparse(self.ui.warn, inst)
             self.ui.flush()
             self.cresult.write('exit 255')
             return
         newhash = hashstate.fromui(lui, self.hashstate.mtimepaths)
         insts = []
         if newhash.mtimehash != self.hashstate.mtimehash:
             addr = _hashaddress(self.baseaddress, self.hashstate.confighash)
             insts.append('unlink %s' % addr)
             # mtimehash is empty if one or more extensions fail to load.
             # to be compatible with hg, still serve the client this time.
             if self.hashstate.mtimehash:
                 insts.append('reconnect')
         if newhash.confighash != self.hashstate.confighash:
             addr = _hashaddress(self.baseaddress, newhash.confighash)
             insts.append('redirect %s' % addr)
         _log('validate: %s\n' % insts)
         self.cresult.write('\0'.join(insts) or '\0')
 
     def chdir(self):
         """Change current directory
 
         Note that the behavior of --cwd option is bit different from this.
         It does not affect --config parameter.
         """
         path = self._readstr()
         if not path:
             return
         _log('chdir to %r\n' % path)
         os.chdir(path)
 
     def setumask(self):
         """Change umask"""
         mask = struct.unpack('>I', self._read(4))[0]
         _log('setumask %r\n' % mask)
         os.umask(mask)
 
     def getpager(self):
         """Read cmdargs and write pager command to r-channel if enabled
 
         If pager isn't enabled, this writes '\0' because channeledoutput
         does not allow to write empty data.
         """
         args = self._readlist()
         try:
             cmd, _func, args, options, _cmdoptions = dispatch._parse(self.ui,
                                                                      args)
         except (error.Abort, error.AmbiguousCommand, error.CommandError,
                 error.UnknownCommand):
             cmd = None
             options = {}
         if not cmd or 'pager' not in options:
             self.cresult.write('\0')
             return
 
         pagercmd = _setuppagercmd(self.ui, options, cmd)
         if pagercmd:
+            # Python's SIGPIPE is SIG_IGN by default. change to SIG_DFL so
+            # we can exit if the pipe to the pager is closed
+            if util.safehasattr(signal, 'SIGPIPE') and \
+                    signal.getsignal(signal.SIGPIPE) == signal.SIG_IGN:
+                signal.signal(signal.SIGPIPE, signal.SIG_DFL)
             self.cresult.write(pagercmd)
         else:
             self.cresult.write('\0')
 
     def setenv(self):
         """Clear and update os.environ
 
         Note that not all variables can make an effect on the running process.
         """
         l = self._readlist()
         try:
             newenv = dict(s.split('=', 1) for s in l)
         except ValueError:
             raise ValueError('unexpected value in setenv request')
         _log('setenv: %r\n' % sorted(newenv.keys()))
         os.environ.clear()
         os.environ.update(newenv)
-        _clearenvaliases(commands.table)
 
     capabilities = commandserver.server.capabilities.copy()
     capabilities.update({'attachio': attachio,
                          'chdir': chdir,
                          'getpager': getpager,
                          'setenv': setenv,
                          'setumask': setumask})
 
-# copied from mercurial/commandserver.py
-class _requesthandler(SocketServer.StreamRequestHandler):
-    def handle(self):
-        # use a different process group from the master process, making this
-        # process pass kernel "is_current_pgrp_orphaned" check so signals like
-        # SIGTSTP, SIGTTIN, SIGTTOU are not ignored.
-        os.setpgid(0, 0)
-        # change random state otherwise forked request handlers would have a
-        # same state inherited from parent.
-        random.seed()
-        ui = self.server.ui
-        repo = self.server.repo
-        sv = None
-        try:
-            sv = chgcmdserver(ui, repo, self.rfile, self.wfile, self.connection,
-                              self.server.hashstate, self.server.baseaddress)
-            try:
-                sv.serve()
-            # handle exceptions that may be raised by command server. most of
-            # known exceptions are caught by dispatch.
-            except error.Abort as inst:
-                ui.warn(_('abort: %s\n') % inst)
-            except IOError as inst:
-                if inst.errno != errno.EPIPE:
-                    raise
-            except KeyboardInterrupt:
-                pass
-            finally:
-                sv.cleanup()
-        except: # re-raises
-            # also write traceback to error channel. otherwise client cannot
-            # see it because it is written to server's stderr by default.
-            if sv:
-                cerr = sv.cerr
-            else:
-                cerr = commandserver.channeledoutput(self.wfile, 'e')
-            traceback.print_exc(file=cerr)
-            raise
-        finally:
-            # trigger __del__ since ForkingMixIn uses os._exit
-            gc.collect()
-
 def _tempaddress(address):
     return '%s.%d.tmp' % (address, os.getpid())
 
 def _hashaddress(address, hashstr):
     return '%s-%s' % (address, hashstr)
 
-class AutoExitMixIn:  # use old-style to comply with SocketServer design
-    lastactive = time.time()
-    idletimeout = 3600  # default 1 hour
+class chgunixservicehandler(object):
+    """Set of operations for chg services"""
+
+    pollinterval = 1  # [sec]
 
-    def startautoexitthread(self):
-        # note: the auto-exit check here is cheap enough to not use a thread,
-        # be done in serve_forever. however SocketServer is hook-unfriendly,
-        # you simply cannot hook serve_forever without copying a lot of code.
-        # besides, serve_forever's docstring suggests using thread.
-        thread = threading.Thread(target=self._autoexitloop)
-        thread.daemon = True
-        thread.start()
+    def __init__(self, ui):
+        self.ui = ui
+        self._idletimeout = ui.configint('chgserver', 'idletimeout', 3600)
+        self._lastactive = time.time()
+
+    def bindsocket(self, sock, address):
+        self._inithashstate(address)
+        self._checkextensions()
+        self._bind(sock)
+        self._createsymlink()
 
-    def _autoexitloop(self, interval=1):
-        while True:
-            time.sleep(interval)
-            if not self.issocketowner():
-                _log('%s is not owned, exiting.\n' % self.server_address)
-                break
-            if time.time() - self.lastactive > self.idletimeout:
-                _log('being idle too long. exiting.\n')
-                break
-        self.shutdown()
+    def _inithashstate(self, address):
+        self._baseaddress = address
+        if self.ui.configbool('chgserver', 'skiphash', False):
+            self._hashstate = None
+            self._realaddress = address
+            return
+        self._hashstate = hashstate.fromui(self.ui)
+        self._realaddress = _hashaddress(address, self._hashstate.confighash)
 
-    def process_request(self, request, address):
-        self.lastactive = time.time()
-        return SocketServer.ForkingMixIn.process_request(
-            self, request, address)
+    def _checkextensions(self):
+        if not self._hashstate:
+            return
+        if extensions.notloaded():
+            # one or more extensions failed to load. mtimehash becomes
+            # meaningless because we do not know the paths of those extensions.
+            # set mtimehash to an illegal hash value to invalidate the server.
+            self._hashstate.mtimehash = ''
 
-    def server_bind(self):
+    def _bind(self, sock):
         # use a unique temp address so we can stat the file and do ownership
         # check later
-        tempaddress = _tempaddress(self.server_address)
-        # use relative path instead of full path at bind() if possible, since
-        # AF_UNIX path has very small length limit (107 chars) on common
-        # platforms (see sys/un.h)
-        dirname, basename = os.path.split(tempaddress)
-        bakwdfd = None
-        if dirname:
-            bakwdfd = os.open('.', os.O_DIRECTORY)
-            os.chdir(dirname)
-        self.socket.bind(basename)
-        self._socketstat = os.stat(basename)
+        tempaddress = _tempaddress(self._realaddress)
+        util.bindunixsocket(sock, tempaddress)
+        self._socketstat = os.stat(tempaddress)
         # rename will replace the old socket file if exists atomically. the
         # old server will detect ownership change and exit.
-        util.rename(basename, self.server_address)
-        if bakwdfd:
-            os.fchdir(bakwdfd)
-            os.close(bakwdfd)
+        util.rename(tempaddress, self._realaddress)
 
-    def issocketowner(self):
+    def _createsymlink(self):
+        if self._baseaddress == self._realaddress:
+            return
+        tempaddress = _tempaddress(self._baseaddress)
+        os.symlink(os.path.basename(self._realaddress), tempaddress)
+        util.rename(tempaddress, self._baseaddress)
+
+    def _issocketowner(self):
         try:
-            stat = os.stat(self.server_address)
+            stat = os.stat(self._realaddress)
             return (stat.st_ino == self._socketstat.st_ino and
                     stat.st_mtime == self._socketstat.st_mtime)
         except OSError:
             return False
 
-    def unlinksocketfile(self):
-        if not self.issocketowner():
+    def unlinksocket(self, address):
+        if not self._issocketowner():
             return
         # it is possible to have a race condition here that we may
         # remove another server's socket file. but that's okay
         # since that server will detect and exit automatically and
         # the client will start a new server on demand.
         try:
-            os.unlink(self.server_address)
+            os.unlink(self._realaddress)
         except OSError as exc:
             if exc.errno != errno.ENOENT:
                 raise
 
-class chgunixservice(commandserver.unixservice):
-    def init(self):
-        if self.repo:
-            # one chgserver can serve multiple repos. drop repo infomation
-            self.ui.setconfig('bundle', 'mainreporoot', '', 'repo')
-            self.repo = None
-        self._inithashstate()
-        self._checkextensions()
-        class cls(AutoExitMixIn, SocketServer.ForkingMixIn,
-                  SocketServer.UnixStreamServer):
-            ui = self.ui
-            repo = self.repo
-            hashstate = self.hashstate
-            baseaddress = self.baseaddress
-        self.server = cls(self.address, _requesthandler)
-        self.server.idletimeout = self.ui.configint(
-            'chgserver', 'idletimeout', self.server.idletimeout)
-        self.server.startautoexitthread()
-        self._createsymlink()
+    def printbanner(self, address):
+        # no "listening at" message should be printed to simulate hg behavior
+        pass
+
+    def shouldexit(self):
+        if not self._issocketowner():
+            self.ui.debug('%s is not owned, exiting.\n' % self._realaddress)
+            return True
+        if time.time() - self._lastactive > self._idletimeout:
+            self.ui.debug('being idle too long. exiting.\n')
+            return True
+        return False
 
-    def _inithashstate(self):
-        self.baseaddress = self.address
-        if self.ui.configbool('chgserver', 'skiphash', False):
-            self.hashstate = None
-            return
-        self.hashstate = hashstate.fromui(self.ui)
-        self.address = _hashaddress(self.address, self.hashstate.confighash)
+    def newconnection(self):
+        self._lastactive = time.time()
+
+    def createcmdserver(self, repo, conn, fin, fout):
+        return chgcmdserver(self.ui, repo, fin, fout, conn,
+                            self._hashstate, self._baseaddress)
 
-    def _checkextensions(self):
-        if not self.hashstate:
-            return
-        if extensions.notloaded():
-            # one or more extensions failed to load. mtimehash becomes
-            # meaningless because we do not know the paths of those extensions.
-            # set mtimehash to an illegal hash value to invalidate the server.
-            self.hashstate.mtimehash = ''
-
-    def _createsymlink(self):
-        if self.baseaddress == self.address:
-            return
-        tempaddress = _tempaddress(self.baseaddress)
-        os.symlink(os.path.basename(self.address), tempaddress)
-        util.rename(tempaddress, self.baseaddress)
-
-    def run(self):
-        try:
-            self.server.serve_forever()
-        finally:
-            self.server.unlinksocketfile()
+def chgunixservice(ui, repo, opts):
+    if repo:
+        # one chgserver can serve multiple repos. drop repo infomation
+        ui.setconfig('bundle', 'mainreporoot', '', 'repo')
+    h = chgunixservicehandler(ui)
+    return commandserver.unixforkingservice(ui, repo=None, opts=opts, handler=h)
 
 def uisetup(ui):
     commandserver._servicemap['chgunix'] = chgunixservice
 
     # CHGINTERNALMARK is temporarily set by chg client to detect if chg will
     # start another chg. drop it to avoid possible side effects.
     if 'CHGINTERNALMARK' in os.environ:
         del os.environ['CHGINTERNALMARK']

hgext/color.py

@@ -1,664 +1,665
 # color.py color output for Mercurial commands
 #
 # Copyright (C) 2007 Kevin Christen <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''colorize output from some commands
 
 The color extension colorizes output from several Mercurial commands.
 For example, the diff command shows additions in green and deletions
 in red, while the status command shows modified files in magenta. Many
 other commands have analogous colors. It is possible to customize
 these colors.
 
 Effects
 -------
 
 Other effects in addition to color, like bold and underlined text, are
 also available. By default, the terminfo database is used to find the
 terminal codes used to change color and effect.  If terminfo is not
 available, then effects are rendered with the ECMA-48 SGR control
 function (aka ANSI escape codes).
 
 The available effects in terminfo mode are 'blink', 'bold', 'dim',
 'inverse', 'invisible', 'italic', 'standout', and 'underline'; in
 ECMA-48 mode, the options are 'bold', 'inverse', 'italic', and
 'underline'.  How each is rendered depends on the terminal emulator.
 Some may not be available for a given terminal type, and will be
 silently ignored.
 
 Labels
 ------
 
 Text receives color effects depending on the labels that it has. Many
 default Mercurial commands emit labelled text. You can also define
 your own labels in templates using the label function, see :hg:`help
 templates`. A single portion of text may have more than one label. In
 that case, effects given to the last label will override any other
 effects. This includes the special "none" effect, which nullifies
 other effects.
 
 Labels are normally invisible. In order to see these labels and their
 position in the text, use the global --color=debug option. The same
 anchor text may be associated to multiple labels, e.g.
 
   [log.changeset changeset.secret|changeset:   22611:6f0a53c8f587]
 
 The following are the default effects for some default labels. Default
 effects may be overridden from your configuration file::
 
   [color]
   status.modified = blue bold underline red_background
   status.added = green bold
   status.removed = red bold blue_background
   status.deleted = cyan bold underline
   status.unknown = magenta bold underline
   status.ignored = black bold
 
   # 'none' turns off all effects
   status.clean = none
   status.copied = none
 
   qseries.applied = blue bold underline
   qseries.unapplied = black bold
   qseries.missing = red bold
 
   diff.diffline = bold
   diff.extended = cyan bold
   diff.file_a = red bold
   diff.file_b = green bold
   diff.hunk = magenta
   diff.deleted = red
   diff.inserted = green
   diff.changed = white
   diff.tab =
   diff.trailingwhitespace = bold red_background
 
   # Blank so it inherits the style of the surrounding label
   changeset.public =
   changeset.draft =
   changeset.secret =
 
   resolve.unresolved = red bold
   resolve.resolved = green bold
 
   bookmarks.active = green
 
   branches.active = none
   branches.closed = black bold
   branches.current = green
   branches.inactive = none
 
   tags.normal = green
   tags.local = black bold
 
   rebase.rebased = blue
   rebase.remaining = red bold
 
   shelve.age = cyan
   shelve.newest = green bold
   shelve.name = blue bold
 
   histedit.remaining = red bold
 
 Custom colors
 -------------
 
 Because there are only eight standard colors, this module allows you
 to define color names for other color slots which might be available
 for your terminal type, assuming terminfo mode.  For instance::
 
   color.brightblue = 12
   color.pink = 207
   color.orange = 202
 
 to set 'brightblue' to color slot 12 (useful for 16 color terminals
 that have brighter colors defined in the upper eight) and, 'pink' and
 'orange' to colors in 256-color xterm's default color cube.  These
 defined colors may then be used as any of the pre-defined eight,
 including appending '_background' to set the background to that color.
 
 Modes
 -----
 
 By default, the color extension will use ANSI mode (or win32 mode on
 Windows) if it detects a terminal. To override auto mode (to enable
 terminfo mode, for example), set the following configuration option::
 
   [color]
   mode = terminfo
 
 Any value other than 'ansi', 'win32', 'terminfo', or 'auto' will
 disable color.
 
 Note that on some systems, terminfo mode may cause problems when using
 color with the pager extension and less -R. less with the -R option
 will only display ECMA-48 color codes, and terminfo mode may sometimes
 emit codes that less doesn't understand. You can work around this by
 either using ansi mode (or auto mode), or by using less -r (which will
 pass through all terminal control codes, not just color control
 codes).
 
 On some systems (such as MSYS in Windows), the terminal may support
 a different color mode than the pager (activated via the "pager"
 extension). It is possible to define separate modes depending on whether
 the pager is active::
 
   [color]
   mode = auto
   pagermode = ansi
 
 If ``pagermode`` is not defined, the ``mode`` will be used.
 '''
 
 from __future__ import absolute_import
 
 import os
+
+from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     commands,
     dispatch,
     extensions,
     subrepo,
     ui as uimod,
     util,
 )
-from mercurial.i18n import _
 
 cmdtable = {}
 command = cmdutil.command(cmdtable)
 # Note for extension authors: ONLY specify testedwith = 'internal' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'internal'
 
 # start and stop parameters for effects
 _effects = {'none': 0, 'black': 30, 'red': 31, 'green': 32, 'yellow': 33,
             'blue': 34, 'magenta': 35, 'cyan': 36, 'white': 37, 'bold': 1,
             'italic': 3, 'underline': 4, 'inverse': 7, 'dim': 2,
             'black_background': 40, 'red_background': 41,
             'green_background': 42, 'yellow_background': 43,
             'blue_background': 44, 'purple_background': 45,
             'cyan_background': 46, 'white_background': 47}
 
 def _terminfosetup(ui, mode):
     '''Initialize terminfo data and the terminal if we're in terminfo mode.'''
 
     global _terminfo_params
     # If we failed to load curses, we go ahead and return.
     if not _terminfo_params:
         return
     # Otherwise, see what the config file says.
     if mode not in ('auto', 'terminfo'):
         return
 
     _terminfo_params.update((key[6:], (False, int(val)))
         for key, val in ui.configitems('color')
         if key.startswith('color.'))
 
     try:
         curses.setupterm()
     except curses.error as e:
         _terminfo_params = {}
         return
 
     for key, (b, e) in _terminfo_params.items():
         if not b:
             continue
         if not curses.tigetstr(e):
             # Most terminals don't support dim, invis, etc, so don't be
             # noisy and use ui.debug().
             ui.debug("no terminfo entry for %s\n" % e)
             del _terminfo_params[key]
     if not curses.tigetstr('setaf') or not curses.tigetstr('setab'):
         # Only warn about missing terminfo entries if we explicitly asked for
         # terminfo mode.
         if mode == "terminfo":
             ui.warn(_("no terminfo entry for setab/setaf: reverting to "
               "ECMA-48 color\n"))
         _terminfo_params = {}
 
 def _modesetup(ui, coloropt):
     global _terminfo_params
 
     if coloropt == 'debug':
         return 'debug'
 
     auto = (coloropt == 'auto')
     always = not auto and util.parsebool(coloropt)
     if not always and not auto:
         return None
 
     formatted = always or (os.environ.get('TERM') != 'dumb' and ui.formatted())
 
     mode = ui.config('color', 'mode', 'auto')
 
     # If pager is active, color.pagermode overrides color.mode.
     if getattr(ui, 'pageractive', False):
         mode = ui.config('color', 'pagermode', mode)
 
     realmode = mode
     if mode == 'auto':
         if os.name == 'nt':
             term = os.environ.get('TERM')
             # TERM won't be defined in a vanilla cmd.exe environment.
 
             # UNIX-like environments on Windows such as Cygwin and MSYS will
             # set TERM. They appear to make a best effort attempt at setting it
             # to something appropriate. However, not all environments with TERM
             # defined support ANSI. Since "ansi" could result in terminal
             # gibberish, we error on the side of selecting "win32". However, if
             # w32effects is not defined, we almost certainly don't support
             # "win32", so don't even try.
             if (term and 'xterm' in term) or not w32effects:
                 realmode = 'ansi'
             else:
                 realmode = 'win32'
         else:
             realmode = 'ansi'
 
     def modewarn():
         # only warn if color.mode was explicitly set and we're in
         # an interactive terminal
         if mode == realmode and ui.interactive():
             ui.warn(_('warning: failed to set color mode to %s\n') % mode)
 
     if realmode == 'win32':
         _terminfo_params = {}
         if not w32effects:
             modewarn()
             return None
         _effects.update(w32effects)
     elif realmode == 'ansi':
         _terminfo_params = {}
     elif realmode == 'terminfo':
         _terminfosetup(ui, mode)
         if not _terminfo_params:
             ## FIXME Shouldn't we return None in this case too?
             modewarn()
             realmode = 'ansi'
     else:
         return None
 
     if always or (auto and formatted):
         return realmode
     return None
 
 try:
     import curses
     # Mapping from effect name to terminfo attribute name or color number.
     # This will also force-load the curses module.
     _terminfo_params = {'none': (True, 'sgr0'),
                         'standout': (True, 'smso'),
                         'underline': (True, 'smul'),
                         'reverse': (True, 'rev'),
                         'inverse': (True, 'rev'),
                         'blink': (True, 'blink'),
                         'dim': (True, 'dim'),
                         'bold': (True, 'bold'),
                         'invisible': (True, 'invis'),
                         'italic': (True, 'sitm'),
                         'black': (False, curses.COLOR_BLACK),
                         'red': (False, curses.COLOR_RED),
                         'green': (False, curses.COLOR_GREEN),
                         'yellow': (False, curses.COLOR_YELLOW),
                         'blue': (False, curses.COLOR_BLUE),
                         'magenta': (False, curses.COLOR_MAGENTA),
                         'cyan': (False, curses.COLOR_CYAN),
                         'white': (False, curses.COLOR_WHITE)}
 except ImportError:
     _terminfo_params = {}
 
 _styles = {'grep.match': 'red bold',
            'grep.linenumber': 'green',
            'grep.rev': 'green',
            'grep.change': 'green',
            'grep.sep': 'cyan',
            'grep.filename': 'magenta',
            'grep.user': 'magenta',
            'grep.date': 'magenta',
            'bookmarks.active': 'green',
            'branches.active': 'none',
            'branches.closed': 'black bold',
            'branches.current': 'green',
            'branches.inactive': 'none',
            'diff.changed': 'white',
            'diff.deleted': 'red',
            'diff.diffline': 'bold',
            'diff.extended': 'cyan bold',
            'diff.file_a': 'red bold',
            'diff.file_b': 'green bold',
            'diff.hunk': 'magenta',
            'diff.inserted': 'green',
            'diff.tab': '',
            'diff.trailingwhitespace': 'bold red_background',
            'changeset.public' : '',
            'changeset.draft' : '',
            'changeset.secret' : '',
            'diffstat.deleted': 'red',
            'diffstat.inserted': 'green',
            'histedit.remaining': 'red bold',
            'ui.prompt': 'yellow',
            'log.changeset': 'yellow',
            'patchbomb.finalsummary': '',
            'patchbomb.from': 'magenta',
            'patchbomb.to': 'cyan',
            'patchbomb.subject': 'green',
            'patchbomb.diffstats': '',
            'rebase.rebased': 'blue',
            'rebase.remaining': 'red bold',
            'resolve.resolved': 'green bold',
            'resolve.unresolved': 'red bold',
            'shelve.age': 'cyan',
            'shelve.newest': 'green bold',
            'shelve.name': 'blue bold',
            'status.added': 'green bold',
            'status.clean': 'none',
            'status.copied': 'none',
            'status.deleted': 'cyan bold underline',
            'status.ignored': 'black bold',
            'status.modified': 'blue bold',
            'status.removed': 'red bold',
            'status.unknown': 'magenta bold underline',
            'tags.normal': 'green',
            'tags.local': 'black bold'}
 
 
 def _effect_str(effect):
     '''Helper function for render_effects().'''
 
     bg = False
     if effect.endswith('_background'):
         bg = True
         effect = effect[:-11]
     attr, val = _terminfo_params[effect]
     if attr:
         return curses.tigetstr(val)
     elif bg:
         return curses.tparm(curses.tigetstr('setab'), val)
     else:
         return curses.tparm(curses.tigetstr('setaf'), val)
 
 def render_effects(text, effects):
     'Wrap text in commands to turn on each effect.'
     if not text:
         return text
     if not _terminfo_params:
         start = [str(_effects[e]) for e in ['none'] + effects.split()]
         start = '\033[' + ';'.join(start) + 'm'
         stop = '\033[' + str(_effects['none']) + 'm'
     else:
         start = ''.join(_effect_str(effect)
                         for effect in ['none'] + effects.split())
         stop = _effect_str('none')
     return ''.join([start, text, stop])
 
 def extstyles():
     for name, ext in extensions.extensions():
         _styles.update(getattr(ext, 'colortable', {}))
 
 def valideffect(effect):
     'Determine if the effect is valid or not.'
     good = False
     if not _terminfo_params and effect in _effects:
         good = True
     elif effect in _terminfo_params or effect[:-11] in _terminfo_params:
         good = True
     return good
 
 def configstyles(ui):
     for status, cfgeffects in ui.configitems('color'):
         if '.' not in status or status.startswith('color.'):
             continue
         cfgeffects = ui.configlist('color', status)
         if cfgeffects:
             good = []
             for e in cfgeffects:
                 if valideffect(e):
                     good.append(e)
                 else:
                     ui.warn(_("ignoring unknown color/effect %r "
                               "(configured in color.%s)\n")
                             % (e, status))
             _styles[status] = ' '.join(good)
 
 class colorui(uimod.ui):
     _colormode = 'ansi'
     def write(self, *args, **opts):
         if self._colormode is None:
             return super(colorui, self).write(*args, **opts)
 
         label = opts.get('label', '')
         if self._buffers and not opts.get('prompt', False):
             if self._bufferapplylabels:
                 self._buffers[-1].extend(self.label(a, label) for a in args)
             else:
                 self._buffers[-1].extend(args)
         elif self._colormode == 'win32':
             for a in args:
                 win32print(a, super(colorui, self).write, **opts)
         else:
             return super(colorui, self).write(
                 *[self.label(a, label) for a in args], **opts)
 
     def write_err(self, *args, **opts):
         if self._colormode is None:
             return super(colorui, self).write_err(*args, **opts)
 
         label = opts.get('label', '')
         if self._bufferstates and self._bufferstates[-1][0]:
             return self.write(*args, **opts)
         if self._colormode == 'win32':
             for a in args:
                 win32print(a, super(colorui, self).write_err, **opts)
         else:
             return super(colorui, self).write_err(
                 *[self.label(a, label) for a in args], **opts)
 
     def showlabel(self, msg, label):
         if label and msg:
             if msg[-1] == '\n':
                 return "[%s|%s]\n" % (label, msg[:-1])
             else:
                 return "[%s|%s]" % (label, msg)
         else:
             return msg
 
     def label(self, msg, label):
         if self._colormode is None:
             return super(colorui, self).label(msg, label)
 
         if self._colormode == 'debug':
             return self.showlabel(msg, label)
 
         effects = []
         for l in label.split():
             s = _styles.get(l, '')
             if s:
                 effects.append(s)
             elif valideffect(l):
                 effects.append(l)
         effects = ' '.join(effects)
         if effects:
             return '\n'.join([render_effects(s, effects)
                               for s in msg.split('\n')])
         return msg
 
 def uisetup(ui):
     if ui.plain():
         return
     if not isinstance(ui, colorui):
         colorui.__bases__ = (ui.__class__,)
         ui.__class__ = colorui
     def colorcmd(orig, ui_, opts, cmd, cmdfunc):
         mode = _modesetup(ui_, opts['color'])
         colorui._colormode = mode
         if mode and mode != 'debug':
             extstyles()
             configstyles(ui_)
         return orig(ui_, opts, cmd, cmdfunc)
     def colorgit(orig, gitsub, commands, env=None, stream=False, cwd=None):
         if gitsub.ui._colormode and len(commands) and commands[0] == "diff":
                 # insert the argument in the front,
                 # the end of git diff arguments is used for paths
                 commands.insert(1, '--color')
         return orig(gitsub, commands, env, stream, cwd)
     extensions.wrapfunction(dispatch, '_runcommand', colorcmd)
     extensions.wrapfunction(subrepo.gitsubrepo, '_gitnodir', colorgit)
 
 def extsetup(ui):
     commands.globalopts.append(
         ('', 'color', 'auto',
          # i18n: 'always', 'auto', 'never', and 'debug' are keywords
          # and should not be translated
          _("when to colorize (boolean, always, auto, never, or debug)"),
          _('TYPE')))
 
 @command('debugcolor', [], 'hg debugcolor')
 def debugcolor(ui, repo, **opts):
     global _styles
     _styles = {}
     for effect in _effects.keys():
         _styles[effect] = effect
     ui.write(('color mode: %s\n') % ui._colormode)
     ui.write(_('available colors:\n'))
     for label, colors in _styles.items():
         ui.write(('%s\n') % colors, label=label)
 
 if os.name != 'nt':
     w32effects = None
 else:
     import ctypes
     import re
 
     _kernel32 = ctypes.windll.kernel32
 
     _WORD = ctypes.c_ushort
 
     _INVALID_HANDLE_VALUE = -1
 
     class _COORD(ctypes.Structure):
         _fields_ = [('X', ctypes.c_short),
                     ('Y', ctypes.c_short)]
 
     class _SMALL_RECT(ctypes.Structure):
         _fields_ = [('Left', ctypes.c_short),
                     ('Top', ctypes.c_short),
                     ('Right', ctypes.c_short),
                     ('Bottom', ctypes.c_short)]
 
     class _CONSOLE_SCREEN_BUFFER_INFO(ctypes.Structure):
         _fields_ = [('dwSize', _COORD),
                     ('dwCursorPosition', _COORD),
                     ('wAttributes', _WORD),
                     ('srWindow', _SMALL_RECT),
                     ('dwMaximumWindowSize', _COORD)]
 
     _STD_OUTPUT_HANDLE = 0xfffffff5L # (DWORD)-11
     _STD_ERROR_HANDLE = 0xfffffff4L  # (DWORD)-12
 
     _FOREGROUND_BLUE = 0x0001
     _FOREGROUND_GREEN = 0x0002
     _FOREGROUND_RED = 0x0004
     _FOREGROUND_INTENSITY = 0x0008
 
     _BACKGROUND_BLUE = 0x0010
     _BACKGROUND_GREEN = 0x0020
     _BACKGROUND_RED = 0x0040
     _BACKGROUND_INTENSITY = 0x0080
 
     _COMMON_LVB_REVERSE_VIDEO = 0x4000
     _COMMON_LVB_UNDERSCORE = 0x8000
 
     # http://msdn.microsoft.com/en-us/library/ms682088%28VS.85%29.aspx
     w32effects = {
         'none': -1,
         'black': 0,
         'red': _FOREGROUND_RED,
         'green': _FOREGROUND_GREEN,
         'yellow': _FOREGROUND_RED | _FOREGROUND_GREEN,
         'blue': _FOREGROUND_BLUE,
         'magenta': _FOREGROUND_BLUE | _FOREGROUND_RED,
         'cyan': _FOREGROUND_BLUE | _FOREGROUND_GREEN,
         'white': _FOREGROUND_RED | _FOREGROUND_GREEN | _FOREGROUND_BLUE,
         'bold': _FOREGROUND_INTENSITY,
         'black_background': 0x100,                  # unused value > 0x0f
         'red_background': _BACKGROUND_RED,
         'green_background': _BACKGROUND_GREEN,
         'yellow_background': _BACKGROUND_RED | _BACKGROUND_GREEN,
         'blue_background': _BACKGROUND_BLUE,
         'purple_background': _BACKGROUND_BLUE | _BACKGROUND_RED,
         'cyan_background': _BACKGROUND_BLUE | _BACKGROUND_GREEN,
         'white_background': (_BACKGROUND_RED | _BACKGROUND_GREEN |
                              _BACKGROUND_BLUE),
         'bold_background': _BACKGROUND_INTENSITY,
         'underline': _COMMON_LVB_UNDERSCORE,  # double-byte charsets only
         'inverse': _COMMON_LVB_REVERSE_VIDEO, # double-byte charsets only
     }
 
     passthrough = set([_FOREGROUND_INTENSITY,
                        _BACKGROUND_INTENSITY,
                        _COMMON_LVB_UNDERSCORE,
                        _COMMON_LVB_REVERSE_VIDEO])
 
     stdout = _kernel32.GetStdHandle(
                   _STD_OUTPUT_HANDLE)  # don't close the handle returned
     if stdout is None or stdout == _INVALID_HANDLE_VALUE:
         w32effects = None
     else:
         csbi = _CONSOLE_SCREEN_BUFFER_INFO()
         if not _kernel32.GetConsoleScreenBufferInfo(
                     stdout, ctypes.byref(csbi)):
             # stdout may not support GetConsoleScreenBufferInfo()
             # when called from subprocess or redirected
             w32effects = None
         else:
             origattr = csbi.wAttributes
             ansire = re.compile('\033\[([^m]*)m([^\033]*)(.*)',
                                 re.MULTILINE | re.DOTALL)
 
     def win32print(text, orig, **opts):
         label = opts.get('label', '')
         attr = origattr
 
         def mapcolor(val, attr):
             if val == -1:
                 return origattr
             elif val in passthrough:
                 return attr | val
             elif val > 0x0f:
                 return (val & 0x70) | (attr & 0x8f)
             else:
                 return (val & 0x07) | (attr & 0xf8)
 
         # determine console attributes based on labels
         for l in label.split():
             style = _styles.get(l, '')
             for effect in style.split():
                 try:
                     attr = mapcolor(w32effects[effect], attr)
                 except KeyError:
                     # w32effects could not have certain attributes so we skip
                     # them if not found
                     pass
         # hack to ensure regexp finds data
         if not text.startswith('\033['):
             text = '\033[m' + text
 
         # Look for ANSI-like codes embedded in text
         m = re.match(ansire, text)
 
         try:
             while m:
                 for sattr in m.group(1).split(';'):
                     if sattr:
                         attr = mapcolor(int(sattr), attr)
                 _kernel32.SetConsoleTextAttribute(stdout, attr)
                 orig(m.group(2), **opts)
                 m = re.match(ansire, m.group(3))
         finally:
             # Explicitly reset original attributes
             _kernel32.SetConsoleTextAttribute(stdout, origattr)

hgext/convert/__init__.py

@@ -1,458 +1,458
 # convert.py Foreign SCM converter
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''import revisions from foreign VCS repositories into Mercurial'''
 
 from __future__ import absolute_import
 
+from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     registrar,
 )
-from mercurial.i18n import _
 
 from . import (
     convcmd,
     cvsps,
     subversion,
 )
 
 cmdtable = {}
 command = cmdutil.command(cmdtable)
 # Note for extension authors: ONLY specify testedwith = 'internal' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'internal'
 
 # Commands definition was moved elsewhere to ease demandload job.
 
 @command('convert',
     [('', 'authors', '',
       _('username mapping filename (DEPRECATED) (use --authormap instead)'),
       _('FILE')),
     ('s', 'source-type', '', _('source repository type'), _('TYPE')),
     ('d', 'dest-type', '', _('destination repository type'), _('TYPE')),
     ('r', 'rev', [], _('import up to source revision REV'), _('REV')),
     ('A', 'authormap', '', _('remap usernames using this file'), _('FILE')),
     ('', 'filemap', '', _('remap file names using contents of file'),
      _('FILE')),
     ('', 'full', None,
      _('apply filemap changes by converting all files again')),
     ('', 'splicemap', '', _('splice synthesized history into place'),
      _('FILE')),
     ('', 'branchmap', '', _('change branch names while converting'),
      _('FILE')),
     ('', 'branchsort', None, _('try to sort changesets by branches')),
     ('', 'datesort', None, _('try to sort changesets by date')),
     ('', 'sourcesort', None, _('preserve source changesets order')),
     ('', 'closesort', None, _('try to reorder closed revisions'))],
    _('hg convert [OPTION]... SOURCE [DEST [REVMAP]]'),
    norepo=True)
 def convert(ui, src, dest=None, revmapfile=None, **opts):
     """convert a foreign SCM repository to a Mercurial one.
 
     Accepted source formats [identifiers]:
 
     - Mercurial [hg]
     - CVS [cvs]
     - Darcs [darcs]
     - git [git]
     - Subversion [svn]
     - Monotone [mtn]
     - GNU Arch [gnuarch]
     - Bazaar [bzr]
     - Perforce [p4]
 
     Accepted destination formats [identifiers]:
 
     - Mercurial [hg]
     - Subversion [svn] (history on branches is not preserved)
 
     If no revision is given, all revisions will be converted.
     Otherwise, convert will only import up to the named revision
     (given in a format understood by the source).
 
     If no destination directory name is specified, it defaults to the
     basename of the source with ``-hg`` appended. If the destination
     repository doesn't exist, it will be created.
 
     By default, all sources except Mercurial will use --branchsort.
     Mercurial uses --sourcesort to preserve original revision numbers
     order. Sort modes have the following effects:
 
     --branchsort  convert from parent to child revision when possible,
                   which means branches are usually converted one after
                   the other. It generates more compact repositories.
 
     --datesort    sort revisions by date. Converted repositories have
                   good-looking changelogs but are often an order of
                   magnitude larger than the same ones generated by
                   --branchsort.
 
     --sourcesort  try to preserve source revisions order, only
                   supported by Mercurial sources.
 
     --closesort   try to move closed revisions as close as possible
                   to parent branches, only supported by Mercurial
                   sources.
 
     If ``REVMAP`` isn't given, it will be put in a default location
     (``<dest>/.hg/shamap`` by default). The ``REVMAP`` is a simple
     text file that maps each source commit ID to the destination ID
     for that revision, like so::
 
       <source ID> <destination ID>
 
     If the file doesn't exist, it's automatically created. It's
     updated on each commit copied, so :hg:`convert` can be interrupted
     and can be run repeatedly to copy new commits.
 
     The authormap is a simple text file that maps each source commit
     author to a destination commit author. It is handy for source SCMs
     that use unix logins to identify authors (e.g.: CVS). One line per
     author mapping and the line format is::
 
       source author = destination author
 
     Empty lines and lines starting with a ``#`` are ignored.
 
     The filemap is a file that allows filtering and remapping of files
     and directories. Each line can contain one of the following
     directives::
 
       include path/to/file-or-dir
 
       exclude path/to/file-or-dir
 
       rename path/to/source path/to/destination
 
     Comment lines start with ``#``. A specified path matches if it
     equals the full relative name of a file or one of its parent
     directories. The ``include`` or ``exclude`` directive with the
     longest matching path applies, so line order does not matter.
 
     The ``include`` directive causes a file, or all files under a
     directory, to be included in the destination repository. The default
     if there are no ``include`` statements is to include everything.
     If there are any ``include`` statements, nothing else is included.
     The ``exclude`` directive causes files or directories to
     be omitted. The ``rename`` directive renames a file or directory if
     it is converted. To rename from a subdirectory into the root of
     the repository, use ``.`` as the path to rename to.
 
     ``--full`` will make sure the converted changesets contain exactly
     the right files with the right content. It will make a full
     conversion of all files, not just the ones that have
     changed. Files that already are correct will not be changed. This
     can be used to apply filemap changes when converting
     incrementally. This is currently only supported for Mercurial and
     Subversion.
 
     The splicemap is a file that allows insertion of synthetic
     history, letting you specify the parents of a revision. This is
     useful if you want to e.g. give a Subversion merge two parents, or
     graft two disconnected series of history together. Each entry
     contains a key, followed by a space, followed by one or two
     comma-separated values::
 
       key parent1, parent2
 
     The key is the revision ID in the source
     revision control system whose parents should be modified (same
     format as a key in .hg/shamap). The values are the revision IDs
     (in either the source or destination revision control system) that
     should be used as the new parents for that node. For example, if
     you have merged "release-1.0" into "trunk", then you should
     specify the revision on "trunk" as the first parent and the one on
     the "release-1.0" branch as the second.
 
     The branchmap is a file that allows you to rename a branch when it is
     being brought in from whatever external repository. When used in
     conjunction with a splicemap, it allows for a powerful combination
     to help fix even the most badly mismanaged repositories and turn them
     into nicely structured Mercurial repositories. The branchmap contains
     lines of the form::
 
       original_branch_name new_branch_name
 
     where "original_branch_name" is the name of the branch in the
     source repository, and "new_branch_name" is the name of the branch
     is the destination repository. No whitespace is allowed in the
     branch names. This can be used to (for instance) move code in one
     repository from "default" to a named branch.
 
     Mercurial Source
     ################
 
     The Mercurial source recognizes the following configuration
     options, which you can set on the command line with ``--config``:
 
     :convert.hg.ignoreerrors: ignore integrity errors when reading.
         Use it to fix Mercurial repositories with missing revlogs, by
         converting from and to Mercurial. Default is False.
 
     :convert.hg.saverev: store original revision ID in changeset
         (forces target IDs to change). It takes a boolean argument and
         defaults to False.
 
     :convert.hg.startrev: specify the initial Mercurial revision.
         The default is 0.
 
     :convert.hg.revs: revset specifying the source revisions to convert.
 
     CVS Source
     ##########
 
     CVS source will use a sandbox (i.e. a checked-out copy) from CVS
     to indicate the starting point of what will be converted. Direct
     access to the repository files is not needed, unless of course the
     repository is ``:local:``. The conversion uses the top level
     directory in the sandbox to find the CVS repository, and then uses
     CVS rlog commands to find files to convert. This means that unless
     a filemap is given, all files under the starting directory will be
     converted, and that any directory reorganization in the CVS
     sandbox is ignored.
 
     The following options can be used with ``--config``:
 
     :convert.cvsps.cache: Set to False to disable remote log caching,
         for testing and debugging purposes. Default is True.
 
     :convert.cvsps.fuzz: Specify the maximum time (in seconds) that is
         allowed between commits with identical user and log message in
         a single changeset. When very large files were checked in as
         part of a changeset then the default may not be long enough.
         The default is 60.
 
     :convert.cvsps.mergeto: Specify a regular expression to which
         commit log messages are matched. If a match occurs, then the
         conversion process will insert a dummy revision merging the
         branch on which this log message occurs to the branch
         indicated in the regex. Default is ``{{mergetobranch
         ([-\\w]+)}}``
 
     :convert.cvsps.mergefrom: Specify a regular expression to which
         commit log messages are matched. If a match occurs, then the
         conversion process will add the most recent revision on the
         branch indicated in the regex as the second parent of the
         changeset. Default is ``{{mergefrombranch ([-\\w]+)}}``
 
     :convert.localtimezone: use local time (as determined by the TZ
         environment variable) for changeset date/times. The default
         is False (use UTC).
 
     :hooks.cvslog: Specify a Python function to be called at the end of
         gathering the CVS log. The function is passed a list with the
         log entries, and can modify the entries in-place, or add or
         delete them.
 
     :hooks.cvschangesets: Specify a Python function to be called after
         the changesets are calculated from the CVS log. The
         function is passed a list with the changeset entries, and can
         modify the changesets in-place, or add or delete them.
 
     An additional "debugcvsps" Mercurial command allows the builtin
     changeset merging code to be run without doing a conversion. Its
     parameters and output are similar to that of cvsps 2.1. Please see
     the command help for more details.
 
     Subversion Source
     #################
 
     Subversion source detects classical trunk/branches/tags layouts.
     By default, the supplied ``svn://repo/path/`` source URL is
     converted as a single branch. If ``svn://repo/path/trunk`` exists
     it replaces the default branch. If ``svn://repo/path/branches``
     exists, its subdirectories are listed as possible branches. If
     ``svn://repo/path/tags`` exists, it is looked for tags referencing
     converted branches. Default ``trunk``, ``branches`` and ``tags``
     values can be overridden with following options. Set them to paths
     relative to the source URL, or leave them blank to disable auto
     detection.
 
     The following options can be set with ``--config``:
 
     :convert.svn.branches: specify the directory containing branches.
         The default is ``branches``.
 
     :convert.svn.tags: specify the directory containing tags. The
         default is ``tags``.
 
     :convert.svn.trunk: specify the name of the trunk branch. The
         default is ``trunk``.
 
     :convert.localtimezone: use local time (as determined by the TZ
         environment variable) for changeset date/times. The default
         is False (use UTC).
 
     Source history can be retrieved starting at a specific revision,
     instead of being integrally converted. Only single branch
     conversions are supported.
 
     :convert.svn.startrev: specify start Subversion revision number.
         The default is 0.
 
     Git Source
     ##########
 
     The Git importer converts commits from all reachable branches (refs
     in refs/heads) and remotes (refs in refs/remotes) to Mercurial.
     Branches are converted to bookmarks with the same name, with the
     leading 'refs/heads' stripped. Git submodules are converted to Git
     subrepos in Mercurial.
 
     The following options can be set with ``--config``:
 
     :convert.git.similarity: specify how similar files modified in a
         commit must be to be imported as renames or copies, as a
         percentage between ``0`` (disabled) and ``100`` (files must be
         identical). For example, ``90`` means that a delete/add pair will
         be imported as a rename if more than 90% of the file hasn't
         changed. The default is ``50``.
 
     :convert.git.findcopiesharder: while detecting copies, look at all
         files in the working copy instead of just changed ones. This
         is very expensive for large projects, and is only effective when
         ``convert.git.similarity`` is greater than 0. The default is False.
 
     :convert.git.remoteprefix: remote refs are converted as bookmarks with
         ``convert.git.remoteprefix`` as a prefix followed by a /. The default
         is 'remote'.
 
     :convert.git.skipsubmodules: does not convert root level .gitmodules files
         or files with 160000 mode indicating a submodule. Default is False.
 
     Perforce Source
     ###############
 
     The Perforce (P4) importer can be given a p4 depot path or a
     client specification as source. It will convert all files in the
     source to a flat Mercurial repository, ignoring labels, branches
     and integrations. Note that when a depot path is given you then
     usually should specify a target directory, because otherwise the
     target may be named ``...-hg``.
 
     The following options can be set with ``--config``:
 
     :convert.p4.encoding: specify the encoding to use when decoding standard
         output of the Perforce command line tool. The default is default system
         encoding.
 
     :convert.p4.startrev: specify initial Perforce revision (a
         Perforce changelist number).
 
     Mercurial Destination
     #####################
 
     The Mercurial destination will recognize Mercurial subrepositories in the
     destination directory, and update the .hgsubstate file automatically if the
     destination subrepositories contain the <dest>/<sub>/.hg/shamap file.
     Converting a repository with subrepositories requires converting a single
     repository at a time, from the bottom up.
 
     .. container:: verbose
 
        An example showing how to convert a repository with subrepositories::
 
          # so convert knows the type when it sees a non empty destination
          $ hg init converted
 
          $ hg convert orig/sub1 converted/sub1
          $ hg convert orig/sub2 converted/sub2
          $ hg convert orig converted
 
     The following options are supported:
 
     :convert.hg.clonebranches: dispatch source branches in separate
         clones. The default is False.
 
     :convert.hg.tagsbranch: branch name for tag revisions, defaults to
         ``default``.
 
     :convert.hg.usebranchnames: preserve branch names. The default is
         True.
 
     :convert.hg.sourcename: records the given string as a 'convert_source' extra
         value on each commit made in the target repository. The default is None.
 
     All Destinations
     ################
 
     All destination types accept the following options:
 
     :convert.skiptags: does not convert tags from the source repo to the target
         repo. The default is False.
     """
     return convcmd.convert(ui, src, dest, revmapfile, **opts)
 
 @command('debugsvnlog', [], 'hg debugsvnlog', norepo=True)
 def debugsvnlog(ui, **opts):
     return subversion.debugsvnlog(ui, **opts)
 
 @command('debugcvsps',
     [
     # Main options shared with cvsps-2.1
     ('b', 'branches', [], _('only return changes on specified branches')),
     ('p', 'prefix', '', _('prefix to remove from file names')),
     ('r', 'revisions', [],
      _('only return changes after or between specified tags')),
     ('u', 'update-cache', None, _("update cvs log cache")),
     ('x', 'new-cache', None, _("create new cvs log cache")),
     ('z', 'fuzz', 60, _('set commit time fuzz in seconds')),
     ('', 'root', '', _('specify cvsroot')),
     # Options specific to builtin cvsps
     ('', 'parents', '', _('show parent changesets')),
     ('', 'ancestors', '', _('show current changeset in ancestor branches')),
     # Options that are ignored for compatibility with cvsps-2.1
     ('A', 'cvs-direct', None, _('ignored for compatibility')),
     ],
     _('hg debugcvsps [OPTION]... [PATH]...'),
     norepo=True)
 def debugcvsps(ui, *args, **opts):
     '''create changeset information from CVS
 
     This command is intended as a debugging tool for the CVS to
     Mercurial converter, and can be used as a direct replacement for
     cvsps.
 
     Hg debugcvsps reads the CVS rlog for current directory (or any
     named directory) in the CVS repository, and converts the log to a
     series of changesets based on matching commit log entries and
     dates.'''
     return cvsps.debugcvsps(ui, *args, **opts)
 
 def kwconverted(ctx, name):
     rev = ctx.extra().get('convert_revision', '')
     if rev.startswith('svn:'):
         if name == 'svnrev':
             return str(subversion.revsplit(rev)[2])
         elif name == 'svnpath':
             return subversion.revsplit(rev)[1]
         elif name == 'svnuuid':
             return subversion.revsplit(rev)[0]
     return rev
 
 templatekeyword = registrar.templatekeyword()
 
 @templatekeyword('svnrev')
 def kwsvnrev(repo, ctx, **args):
     """String. Converted subversion revision number."""
     return kwconverted(ctx, 'svnrev')
 
 @templatekeyword('svnpath')
 def kwsvnpath(repo, ctx, **args):
     """String. Converted subversion revision project path."""
     return kwconverted(ctx, 'svnpath')
 
 @templatekeyword('svnuuid')
 def kwsvnuuid(repo, ctx, **args):
     """String. Converted subversion revision repository identifier."""
     return kwconverted(ctx, 'svnuuid')
 
 # tell hggettext to extract docstrings from these functions:
 i18nfunctions = [kwsvnrev, kwsvnpath, kwsvnuuid]

hgext/convert/bzr.py

@@ -1,295 +1,296
 # bzr.py - bzr support for the convert extension
 #
 #  Copyright 2008, 2009 Marek Kubica <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 # This module is for handling 'bzr', that was formerly known as Bazaar-NG;
 # it cannot access 'bar' repositories, but they were never used very much
 from __future__ import absolute_import
 
 import os
+
+from mercurial.i18n import _
 from mercurial import (
     demandimport,
     error
 )
-from mercurial.i18n import _
 from . import common
 
 # these do not work with demandimport, blacklist
 demandimport.ignore.extend([
         'bzrlib.transactions',
         'bzrlib.urlutils',
         'ElementPath',
     ])
 
 try:
     # bazaar imports
     import bzrlib.bzrdir
     import bzrlib.errors
     import bzrlib.revision
     import bzrlib.revisionspec.RevisionSpec
     bzrdir = bzrlib.bzrdir
     errors = bzrlib.errors
     revision = bzrlib.revision
     revisionspec = bzrlib.revisionspec
 except ImportError:
     pass
 
 supportedkinds = ('file', 'symlink')
 
 class bzr_source(common.converter_source):
     """Reads Bazaar repositories by using the Bazaar Python libraries"""
 
     def __init__(self, ui, path, revs=None):
         super(bzr_source, self).__init__(ui, path, revs=revs)
 
         if not os.path.exists(os.path.join(path, '.bzr')):
             raise common.NoRepo(_('%s does not look like a Bazaar repository')
                               % path)
 
         try:
             # access bzrlib stuff
             bzrdir
         except NameError:
             raise common.NoRepo(_('Bazaar modules could not be loaded'))
 
         path = os.path.abspath(path)
         self._checkrepotype(path)
         try:
             self.sourcerepo = bzrdir.BzrDir.open(path).open_repository()
         except errors.NoRepositoryPresent:
             raise common.NoRepo(_('%s does not look like a Bazaar repository')
                               % path)
         self._parentids = {}
 
     def _checkrepotype(self, path):
         # Lightweight checkouts detection is informational but probably
         # fragile at API level. It should not terminate the conversion.
         try:
             dir = bzrdir.BzrDir.open_containing(path)[0]
             try:
                 tree = dir.open_workingtree(recommend_upgrade=False)
                 branch = tree.branch
             except (errors.NoWorkingTree, errors.NotLocalUrl):
                 tree = None
                 branch = dir.open_branch()
             if (tree is not None and tree.bzrdir.root_transport.base !=
                 branch.bzrdir.root_transport.base):
                 self.ui.warn(_('warning: lightweight checkouts may cause '
                                'conversion failures, try with a regular '
                                'branch instead.\n'))
         except Exception:
             self.ui.note(_('bzr source type could not be determined\n'))
 
     def before(self):
         """Before the conversion begins, acquire a read lock
         for all the operations that might need it. Fortunately
         read locks don't block other reads or writes to the
         repository, so this shouldn't have any impact on the usage of
         the source repository.
 
         The alternative would be locking on every operation that
         needs locks (there are currently two: getting the file and
         getting the parent map) and releasing immediately after,
         but this approach can take even 40% longer."""
         self.sourcerepo.lock_read()
 
     def after(self):
         self.sourcerepo.unlock()
 
     def _bzrbranches(self):
         return self.sourcerepo.find_branches(using=True)
 
     def getheads(self):
         if not self.revs:
             # Set using=True to avoid nested repositories (see issue3254)
             heads = sorted([b.last_revision() for b in self._bzrbranches()])
         else:
             revid = None
             for branch in self._bzrbranches():
                 try:
                     r = revisionspec.RevisionSpec.from_string(self.revs[0])
                     info = r.in_history(branch)
                 except errors.BzrError:
                     pass
                 revid = info.rev_id
             if revid is None:
                 raise error.Abort(_('%s is not a valid revision')
                                   % self.revs[0])
             heads = [revid]
         # Empty repositories return 'null:', which cannot be retrieved
         heads = [h for h in heads if h != 'null:']
         return heads
 
     def getfile(self, name, rev):
         revtree = self.sourcerepo.revision_tree(rev)
         fileid = revtree.path2id(name.decode(self.encoding or 'utf-8'))
         kind = None
         if fileid is not None:
             kind = revtree.kind(fileid)
         if kind not in supportedkinds:
             # the file is not available anymore - was deleted
             return None, None
         mode = self._modecache[(name, rev)]
         if kind == 'symlink':
             target = revtree.get_symlink_target(fileid)
             if target is None:
                 raise error.Abort(_('%s.%s symlink has no target')
                                  % (name, rev))
             return target, mode
         else:
             sio = revtree.get_file(fileid)
             return sio.read(), mode
 
     def getchanges(self, version, full):
         if full:
             raise error.Abort(_("convert from cvs does not support --full"))
         self._modecache = {}
         self._revtree = self.sourcerepo.revision_tree(version)
         # get the parentids from the cache
         parentids = self._parentids.pop(version)
         # only diff against first parent id
         prevtree = self.sourcerepo.revision_tree(parentids[0])
         files, changes = self._gettreechanges(self._revtree, prevtree)
         return files, changes, set()
 
     def getcommit(self, version):
         rev = self.sourcerepo.get_revision(version)
         # populate parent id cache
         if not rev.parent_ids:
             parents = []
             self._parentids[version] = (revision.NULL_REVISION,)
         else:
             parents = self._filterghosts(rev.parent_ids)
             self._parentids[version] = parents
 
         branch = self.recode(rev.properties.get('branch-nick', u'default'))
         if branch == 'trunk':
             branch = 'default'
         return common.commit(parents=parents,
                 date='%d %d' % (rev.timestamp, -rev.timezone),
                 author=self.recode(rev.committer),
                 desc=self.recode(rev.message),
                 branch=branch,
                 rev=version)
 
     def gettags(self):
         bytetags = {}
         for branch in self._bzrbranches():
             if not branch.supports_tags():
                 return {}
             tagdict = branch.tags.get_tag_dict()
             for name, rev in tagdict.iteritems():
                 bytetags[self.recode(name)] = rev
         return bytetags
 
     def getchangedfiles(self, rev, i):
         self._modecache = {}
         curtree = self.sourcerepo.revision_tree(rev)
         if i is not None:
             parentid = self._parentids[rev][i]
         else:
             # no parent id, get the empty revision
             parentid = revision.NULL_REVISION
 
         prevtree = self.sourcerepo.revision_tree(parentid)
         changes = [e[0] for e in self._gettreechanges(curtree, prevtree)[0]]
         return changes
 
     def _gettreechanges(self, current, origin):
         revid = current._revision_id
         changes = []
         renames = {}
         seen = set()
         # Process the entries by reverse lexicographic name order to
         # handle nested renames correctly, most specific first.
         curchanges = sorted(current.iter_changes(origin),
                             key=lambda c: c[1][0] or c[1][1],
                             reverse=True)
         for (fileid, paths, changed_content, versioned, parent, name,
             kind, executable) in curchanges:
 
             if paths[0] == u'' or paths[1] == u'':
                 # ignore changes to tree root
                 continue
 
             # bazaar tracks directories, mercurial does not, so
             # we have to rename the directory contents
             if kind[1] == 'directory':
                 if kind[0] not in (None, 'directory'):
                     # Replacing 'something' with a directory, record it
                     # so it can be removed.
                     changes.append((self.recode(paths[0]), revid))
 
                 if kind[0] == 'directory' and None not in paths:
                     renaming = paths[0] != paths[1]
                     # neither an add nor an delete - a move
                     # rename all directory contents manually
                     subdir = origin.inventory.path2id(paths[0])
                     # get all child-entries of the directory
                     for name, entry in origin.inventory.iter_entries(subdir):
                         # hg does not track directory renames
                         if entry.kind == 'directory':
                             continue
                         frompath = self.recode(paths[0] + '/' + name)
                         if frompath in seen:
                             # Already handled by a more specific change entry
                             # This is important when you have:
                             # a => b
                             # a/c => a/c
                             # Here a/c must not be renamed into b/c
                             continue
                         seen.add(frompath)
                         if not renaming:
                             continue
                         topath = self.recode(paths[1] + '/' + name)
                         # register the files as changed
                         changes.append((frompath, revid))
                         changes.append((topath, revid))
                         # add to mode cache
                         mode = ((entry.executable and 'x')
                                 or (entry.kind == 'symlink' and 's')
                                 or '')
                         self._modecache[(topath, revid)] = mode
                         # register the change as move
                         renames[topath] = frompath
 
                 # no further changes, go to the next change
                 continue
 
             # we got unicode paths, need to convert them
             path, topath = paths
             if path is not None:
                 path = self.recode(path)
             if topath is not None:
                 topath = self.recode(topath)
             seen.add(path or topath)
 
             if topath is None:
                 # file deleted
                 changes.append((path, revid))
                 continue
 
             # renamed
             if path and path != topath:
                 renames[topath] = path
                 changes.append((path, revid))
 
             # populate the mode cache
             kind, executable = [e[1] for e in (kind, executable)]
             mode = ((executable and 'x') or (kind == 'symlink' and 'l')
                     or '')
             self._modecache[(topath, revid)] = mode
             changes.append((topath, revid))
 
         return changes, renames
 
     def _filterghosts(self, ids):
         """Filters out ghost revisions which hg does not support, see
         <http://bazaar-vcs.org/GhostRevision>
         """
         parentmap = self.sourcerepo.get_parent_map(ids)
         parents = tuple([parent for parent in ids if parent in parentmap])
         return parents

hgext/convert/common.py

@@ -1,494 +1,494
 # common.py - common code for the convert extension
 #
 #  Copyright 2005-2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import base64
-import cPickle as pickle
 import datetime
 import errno
 import os
 import re
 import subprocess
 
+from mercurial.i18n import _
 from mercurial import (
     error,
     phases,
     util,
 )
-from mercurial.i18n import _
 
+pickle = util.pickle
 propertycache = util.propertycache
 
 def encodeargs(args):
     def encodearg(s):
         lines = base64.encodestring(s)
         lines = [l.splitlines()[0] for l in lines]
         return ''.join(lines)
 
     s = pickle.dumps(args)
     return encodearg(s)
 
 def decodeargs(s):
     s = base64.decodestring(s)
     return pickle.loads(s)
 
 class MissingTool(Exception):
     pass
 
 def checktool(exe, name=None, abort=True):
     name = name or exe
     if not util.findexe(exe):
         if abort:
             exc = error.Abort
         else:
             exc = MissingTool
         raise exc(_('cannot find required "%s" tool') % name)
 
 class NoRepo(Exception):
     pass
 
 SKIPREV = 'SKIP'
 
 class commit(object):
     def __init__(self, author, date, desc, parents, branch=None, rev=None,
                  extra={}, sortkey=None, saverev=True, phase=phases.draft,
                  optparents=None):
         self.author = author or 'unknown'
         self.date = date or '0 0'
         self.desc = desc
         self.parents = parents # will be converted and used as parents
         self.optparents = optparents or [] # will be used if already converted
         self.branch = branch
         self.rev = rev
         self.extra = extra
         self.sortkey = sortkey
         self.saverev = saverev
         self.phase = phase
 
 class converter_source(object):
     """Conversion source interface"""
 
     def __init__(self, ui, path=None, revs=None):
         """Initialize conversion source (or raise NoRepo("message")
         exception if path is not a valid repository)"""
         self.ui = ui
         self.path = path
         self.revs = revs
 
         self.encoding = 'utf-8'
 
     def checkhexformat(self, revstr, mapname='splicemap'):
         """ fails if revstr is not a 40 byte hex. mercurial and git both uses
             such format for their revision numbering
         """
         if not re.match(r'[0-9a-fA-F]{40,40}$', revstr):
             raise error.Abort(_('%s entry %s is not a valid revision'
                                ' identifier') % (mapname, revstr))
 
     def before(self):
         pass
 
     def after(self):
         pass
 
     def targetfilebelongstosource(self, targetfilename):
         """Returns true if the given targetfile belongs to the source repo. This
         is useful when only a subdirectory of the target belongs to the source
         repo."""
         # For normal full repo converts, this is always True.
         return True
 
     def setrevmap(self, revmap):
         """set the map of already-converted revisions"""
         pass
 
     def getheads(self):
         """Return a list of this repository's heads"""
         raise NotImplementedError
 
     def getfile(self, name, rev):
         """Return a pair (data, mode) where data is the file content
         as a string and mode one of '', 'x' or 'l'. rev is the
         identifier returned by a previous call to getchanges().
         Data is None if file is missing/deleted in rev.
         """
         raise NotImplementedError
 
     def getchanges(self, version, full):
         """Returns a tuple of (files, copies, cleanp2).
 
         files is a sorted list of (filename, id) tuples for all files
         changed between version and its first parent returned by
         getcommit(). If full, all files in that revision is returned.
         id is the source revision id of the file.
 
         copies is a dictionary of dest: source
 
         cleanp2 is the set of files filenames that are clean against p2.
         (Files that are clean against p1 are already not in files (unless
         full). This makes it possible to handle p2 clean files similarly.)
         """
         raise NotImplementedError
 
     def getcommit(self, version):
         """Return the commit object for version"""
         raise NotImplementedError
 
     def numcommits(self):
         """Return the number of commits in this source.
 
         If unknown, return None.
         """
         return None
 
     def gettags(self):
         """Return the tags as a dictionary of name: revision
 
         Tag names must be UTF-8 strings.
         """
         raise NotImplementedError
 
     def recode(self, s, encoding=None):
         if not encoding:
             encoding = self.encoding or 'utf-8'
 
         if isinstance(s, unicode):
             return s.encode("utf-8")
         try:
             return s.decode(encoding).encode("utf-8")
         except UnicodeError:
             try:
                 return s.decode("latin-1").encode("utf-8")
             except UnicodeError:
                 return s.decode(encoding, "replace").encode("utf-8")
 
     def getchangedfiles(self, rev, i):
         """Return the files changed by rev compared to parent[i].
 
         i is an index selecting one of the parents of rev.  The return
         value should be the list of files that are different in rev and
         this parent.
 
         If rev has no parents, i is None.
 
         This function is only needed to support --filemap
         """
         raise NotImplementedError
 
     def converted(self, rev, sinkrev):
         '''Notify the source that a revision has been converted.'''
         pass
 
     def hasnativeorder(self):
         """Return true if this source has a meaningful, native revision
         order. For instance, Mercurial revisions are store sequentially
         while there is no such global ordering with Darcs.
         """
         return False
 
     def hasnativeclose(self):
         """Return true if this source has ability to close branch.
         """
         return False
 
     def lookuprev(self, rev):
         """If rev is a meaningful revision reference in source, return
         the referenced identifier in the same format used by getcommit().
         return None otherwise.
         """
         return None
 
     def getbookmarks(self):
         """Return the bookmarks as a dictionary of name: revision
 
         Bookmark names are to be UTF-8 strings.
         """
         return {}
 
     def checkrevformat(self, revstr, mapname='splicemap'):
         """revstr is a string that describes a revision in the given
            source control system.  Return true if revstr has correct
            format.
         """
         return True
 
 class converter_sink(object):
     """Conversion sink (target) interface"""
 
     def __init__(self, ui, path):
         """Initialize conversion sink (or raise NoRepo("message")
         exception if path is not a valid repository)
 
         created is a list of paths to remove if a fatal error occurs
         later"""
         self.ui = ui
         self.path = path
         self.created = []
 
     def revmapfile(self):
         """Path to a file that will contain lines
         source_rev_id sink_rev_id
         mapping equivalent revision identifiers for each system."""
         raise NotImplementedError
 
     def authorfile(self):
         """Path to a file that will contain lines
         srcauthor=dstauthor
         mapping equivalent authors identifiers for each system."""
         return None
 
     def putcommit(self, files, copies, parents, commit, source, revmap, full,
                   cleanp2):
         """Create a revision with all changed files listed in 'files'
         and having listed parents. 'commit' is a commit object
         containing at a minimum the author, date, and message for this
         changeset.  'files' is a list of (path, version) tuples,
         'copies' is a dictionary mapping destinations to sources,
         'source' is the source repository, and 'revmap' is a mapfile
         of source revisions to converted revisions. Only getfile() and
         lookuprev() should be called on 'source'. 'full' means that 'files'
         is complete and all other files should be removed.
         'cleanp2' is a set of the filenames that are unchanged from p2
         (only in the common merge case where there two parents).
 
         Note that the sink repository is not told to update itself to
         a particular revision (or even what that revision would be)
         before it receives the file data.
         """
         raise NotImplementedError
 
     def puttags(self, tags):
         """Put tags into sink.
 
         tags: {tagname: sink_rev_id, ...} where tagname is an UTF-8 string.
         Return a pair (tag_revision, tag_parent_revision), or (None, None)
         if nothing was changed.
         """
         raise NotImplementedError
 
     def setbranch(self, branch, pbranches):
         """Set the current branch name. Called before the first putcommit
         on the branch.
         branch: branch name for subsequent commits
         pbranches: (converted parent revision, parent branch) tuples"""
         pass
 
     def setfilemapmode(self, active):
         """Tell the destination that we're using a filemap
 
         Some converter_sources (svn in particular) can claim that a file
         was changed in a revision, even if there was no change.  This method
         tells the destination that we're using a filemap and that it should
         filter empty revisions.
         """
         pass
 
     def before(self):
         pass
 
     def after(self):
         pass
 
     def putbookmarks(self, bookmarks):
         """Put bookmarks into sink.
 
         bookmarks: {bookmarkname: sink_rev_id, ...}
         where bookmarkname is an UTF-8 string.
         """
         pass
 
     def hascommitfrommap(self, rev):
         """Return False if a rev mentioned in a filemap is known to not be
         present."""
         raise NotImplementedError
 
     def hascommitforsplicemap(self, rev):
         """This method is for the special needs for splicemap handling and not
         for general use. Returns True if the sink contains rev, aborts on some
         special cases."""
         raise NotImplementedError
 
 class commandline(object):
     def __init__(self, ui, command):
         self.ui = ui
         self.command = command
 
     def prerun(self):
         pass
 
     def postrun(self):
         pass
 
     def _cmdline(self, cmd, *args, **kwargs):
         cmdline = [self.command, cmd] + list(args)
         for k, v in kwargs.iteritems():
             if len(k) == 1:
                 cmdline.append('-' + k)
             else:
                 cmdline.append('--' + k.replace('_', '-'))
             try:
                 if len(k) == 1:
                     cmdline.append('' + v)
                 else:
                     cmdline[-1] += '=' + v
             except TypeError:
                 pass
         cmdline = [util.shellquote(arg) for arg in cmdline]
         if not self.ui.debugflag:
             cmdline += ['2>', os.devnull]
         cmdline = ' '.join(cmdline)
         return cmdline
 
     def _run(self, cmd, *args, **kwargs):
         def popen(cmdline):
             p = subprocess.Popen(cmdline, shell=True, bufsize=-1,
                     close_fds=util.closefds,
                     stdout=subprocess.PIPE)
             return p
         return self._dorun(popen, cmd, *args, **kwargs)
 
     def _run2(self, cmd, *args, **kwargs):
         return self._dorun(util.popen2, cmd, *args, **kwargs)
 
     def _run3(self, cmd, *args, **kwargs):
         return self._dorun(util.popen3, cmd, *args, **kwargs)
 
     def _dorun(self, openfunc, cmd,  *args, **kwargs):
         cmdline = self._cmdline(cmd, *args, **kwargs)
         self.ui.debug('running: %s\n' % (cmdline,))
         self.prerun()
         try:
             return openfunc(cmdline)
         finally:
             self.postrun()
 
     def run(self, cmd, *args, **kwargs):
         p = self._run(cmd, *args, **kwargs)
         output = p.communicate()[0]
         self.ui.debug(output)
         return output, p.returncode
 
     def runlines(self, cmd, *args, **kwargs):
         p = self._run(cmd, *args, **kwargs)
         output = p.stdout.readlines()
         p.wait()
         self.ui.debug(''.join(output))
         return output, p.returncode
 
     def checkexit(self, status, output=''):
         if status:
             if output:
                 self.ui.warn(_('%s error:\n') % self.command)
                 self.ui.warn(output)
             msg = util.explainexit(status)[0]
             raise error.Abort('%s %s' % (self.command, msg))
 
     def run0(self, cmd, *args, **kwargs):
         output, status = self.run(cmd, *args, **kwargs)
         self.checkexit(status, output)
         return output
 
     def runlines0(self, cmd, *args, **kwargs):
         output, status = self.runlines(cmd, *args, **kwargs)
         self.checkexit(status, ''.join(output))
         return output
 
     @propertycache
     def argmax(self):
         # POSIX requires at least 4096 bytes for ARG_MAX
         argmax = 4096
         try:
             argmax = os.sysconf("SC_ARG_MAX")
         except (AttributeError, ValueError):
             pass
 
         # Windows shells impose their own limits on command line length,
         # down to 2047 bytes for cmd.exe under Windows NT/2k and 2500 bytes
         # for older 4nt.exe. See http://support.microsoft.com/kb/830473 for
         # details about cmd.exe limitations.
 
         # Since ARG_MAX is for command line _and_ environment, lower our limit
         # (and make happy Windows shells while doing this).
         return argmax // 2 - 1
 
     def _limit_arglist(self, arglist, cmd, *args, **kwargs):
         cmdlen = len(self._cmdline(cmd, *args, **kwargs))
         limit = self.argmax - cmdlen
         bytes = 0
         fl = []
         for fn in arglist:
             b = len(fn) + 3
             if bytes + b < limit or len(fl) == 0:
                 fl.append(fn)
                 bytes += b
             else:
                 yield fl
                 fl = [fn]
                 bytes = b
         if fl:
             yield fl
 
     def xargs(self, arglist, cmd, *args, **kwargs):
         for l in self._limit_arglist(arglist, cmd, *args, **kwargs):
             self.run0(cmd, *(list(args) + l), **kwargs)
 
 class mapfile(dict):
     def __init__(self, ui, path):
         super(mapfile, self).__init__()
         self.ui = ui
         self.path = path
         self.fp = None
         self.order = []
         self._read()
 
     def _read(self):
         if not self.path:
             return
         try:
             fp = open(self.path, 'r')
         except IOError as err:
             if err.errno != errno.ENOENT:
                 raise
             return
         for i, line in enumerate(fp):
             line = line.splitlines()[0].rstrip()
             if not line:
                 # Ignore blank lines
                 continue
             try:
                 key, value = line.rsplit(' ', 1)
             except ValueError:
                 raise error.Abort(
                     _('syntax error in %s(%d): key/value pair expected')
                     % (self.path, i + 1))
             if key not in self:
                 self.order.append(key)
             super(mapfile, self).__setitem__(key, value)
         fp.close()
 
     def __setitem__(self, key, value):
         if self.fp is None:
             try:
                 self.fp = open(self.path, 'a')
             except IOError as err:
                 raise error.Abort(_('could not open map file %r: %s') %
                                  (self.path, err.strerror))
         self.fp.write('%s %s\n' % (key, value))
         self.fp.flush()
         super(mapfile, self).__setitem__(key, value)
 
     def close(self):
         if self.fp:
             self.fp.close()
             self.fp = None
 
 def makedatetimestamp(t):
     """Like util.makedate() but for time t instead of current time"""
     delta = (datetime.datetime.utcfromtimestamp(t) -
              datetime.datetime.fromtimestamp(t))
     tz = delta.days * 86400 + delta.seconds
     return t, tz

hgext/convert/convcmd.py

@@ -1,611 +1,611
 # convcmd - convert extension commands definition
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import os
 import shlex
 import shutil
 
+from mercurial.i18n import _
 from mercurial import (
     encoding,
     error,
     hg,
     util,
 )
-from mercurial.i18n import _
 
 from . import (
     bzr,
     common,
     cvs,
     darcs,
     filemap,
     git,
     gnuarch,
     hg as hgconvert,
     monotone,
     p4,
     subversion,
 )
 
 mapfile = common.mapfile
 MissingTool = common.MissingTool
 NoRepo = common.NoRepo
 SKIPREV = common.SKIPREV
 
 bzr_source = bzr.bzr_source
 convert_cvs = cvs.convert_cvs
 convert_git = git.convert_git
 darcs_source = darcs.darcs_source
 gnuarch_source = gnuarch.gnuarch_source
 mercurial_sink = hgconvert.mercurial_sink
 mercurial_source = hgconvert.mercurial_source
 monotone_source = monotone.monotone_source
 p4_source = p4.p4_source
 svn_sink = subversion.svn_sink
 svn_source = subversion.svn_source
 
 orig_encoding = 'ascii'
 
 def recode(s):
     if isinstance(s, unicode):
         return s.encode(orig_encoding, 'replace')
     else:
         return s.decode('utf-8').encode(orig_encoding, 'replace')
 
 def mapbranch(branch, branchmap):
     '''
     >>> bmap = {'default': 'branch1'}
     >>> for i in ['', None]:
     ...     mapbranch(i, bmap)
     'branch1'
     'branch1'
     >>> bmap = {'None': 'branch2'}
     >>> for i in ['', None]:
     ...     mapbranch(i, bmap)
     'branch2'
     'branch2'
     >>> bmap = {'None': 'branch3', 'default': 'branch4'}
     >>> for i in ['None', '', None, 'default', 'branch5']:
     ...     mapbranch(i, bmap)
     'branch3'
     'branch4'
     'branch4'
     'branch4'
     'branch5'
     '''
     # If branch is None or empty, this commit is coming from the source
     # repository's default branch and destined for the default branch in the
     # destination repository. For such commits, using a literal "default"
     # in branchmap below allows the user to map "default" to an alternate
     # default branch in the destination repository.
     branch = branchmap.get(branch or 'default', branch)
     # At some point we used "None" literal to denote the default branch,
     # attempt to use that for backward compatibility.
     if (not branch):
         branch = branchmap.get(str(None), branch)
     return branch
 
 source_converters = [
     ('cvs', convert_cvs, 'branchsort'),
     ('git', convert_git, 'branchsort'),
     ('svn', svn_source, 'branchsort'),
     ('hg', mercurial_source, 'sourcesort'),
     ('darcs', darcs_source, 'branchsort'),
     ('mtn', monotone_source, 'branchsort'),
     ('gnuarch', gnuarch_source, 'branchsort'),
     ('bzr', bzr_source, 'branchsort'),
     ('p4', p4_source, 'branchsort'),
     ]
 
 sink_converters = [
     ('hg', mercurial_sink),
     ('svn', svn_sink),
     ]
 
 def convertsource(ui, path, type, revs):
     exceptions = []
     if type and type not in [s[0] for s in source_converters]:
         raise error.Abort(_('%s: invalid source repository type') % type)
     for name, source, sortmode in source_converters:
         try:
             if not type or name == type:
                 return source(ui, path, revs), sortmode
         except (NoRepo, MissingTool) as inst:
             exceptions.append(inst)
     if not ui.quiet:
         for inst in exceptions:
             ui.write("%s\n" % inst)
     raise error.Abort(_('%s: missing or unsupported repository') % path)
 
 def convertsink(ui, path, type):
     if type and type not in [s[0] for s in sink_converters]:
         raise error.Abort(_('%s: invalid destination repository type') % type)
     for name, sink in sink_converters:
         try:
             if not type or name == type:
                 return sink(ui, path)
         except NoRepo as inst:
             ui.note(_("convert: %s\n") % inst)
         except MissingTool as inst:
             raise error.Abort('%s\n' % inst)
     raise error.Abort(_('%s: unknown repository type') % path)
 
 class progresssource(object):
     def __init__(self, ui, source, filecount):
         self.ui = ui
         self.source = source
         self.filecount = filecount
         self.retrieved = 0
 
     def getfile(self, file, rev):
         self.retrieved += 1
         self.ui.progress(_('getting files'), self.retrieved,
                          item=file, total=self.filecount, unit=_('files'))
         return self.source.getfile(file, rev)
 
     def targetfilebelongstosource(self, targetfilename):
         return self.source.targetfilebelongstosource(targetfilename)
 
     def lookuprev(self, rev):
         return self.source.lookuprev(rev)
 
     def close(self):
         self.ui.progress(_('getting files'), None)
 
 class converter(object):
     def __init__(self, ui, source, dest, revmapfile, opts):
 
         self.source = source
         self.dest = dest
         self.ui = ui
         self.opts = opts
         self.commitcache = {}
         self.authors = {}
         self.authorfile = None
 
         # Record converted revisions persistently: maps source revision
         # ID to target revision ID (both strings).  (This is how
         # incremental conversions work.)
         self.map = mapfile(ui, revmapfile)
 
         # Read first the dst author map if any
         authorfile = self.dest.authorfile()
         if authorfile and os.path.exists(authorfile):
             self.readauthormap(authorfile)
         # Extend/Override with new author map if necessary
         if opts.get('authormap'):
             self.readauthormap(opts.get('authormap'))
             self.authorfile = self.dest.authorfile()
 
         self.splicemap = self.parsesplicemap(opts.get('splicemap'))
         self.branchmap = mapfile(ui, opts.get('branchmap'))
 
     def parsesplicemap(self, path):
         """ check and validate the splicemap format and
             return a child/parents dictionary.
             Format checking has two parts.
             1. generic format which is same across all source types
             2. specific format checking which may be different for
                different source type.  This logic is implemented in
                checkrevformat function in source files like
                hg.py, subversion.py etc.
         """
 
         if not path:
             return {}
         m = {}
         try:
             fp = open(path, 'r')
             for i, line in enumerate(fp):
                 line = line.splitlines()[0].rstrip()
                 if not line:
                     # Ignore blank lines
                     continue
                 # split line
                 lex = shlex.shlex(line, posix=True)
                 lex.whitespace_split = True
                 lex.whitespace += ','
                 line = list(lex)
                 # check number of parents
                 if not (2 <= len(line) <= 3):
                     raise error.Abort(_('syntax error in %s(%d): child parent1'
                                        '[,parent2] expected') % (path, i + 1))
                 for part in line:
                     self.source.checkrevformat(part)
                 child, p1, p2 = line[0], line[1:2], line[2:]
                 if p1 == p2:
                     m[child] = p1
                 else:
                     m[child] = p1 + p2
          # if file does not exist or error reading, exit
         except IOError:
             raise error.Abort(_('splicemap file not found or error reading %s:')
                                % path)
         return m
 
 
     def walktree(self, heads):
         '''Return a mapping that identifies the uncommitted parents of every
         uncommitted changeset.'''
         visit = heads
         known = set()
         parents = {}
         numcommits = self.source.numcommits()
         while visit:
             n = visit.pop(0)
             if n in known:
                 continue
             if n in self.map:
                 m = self.map[n]
                 if m == SKIPREV or self.dest.hascommitfrommap(m):
                     continue
             known.add(n)
             self.ui.progress(_('scanning'), len(known), unit=_('revisions'),
                              total=numcommits)
             commit = self.cachecommit(n)
             parents[n] = []
             for p in commit.parents:
                 parents[n].append(p)
                 visit.append(p)
         self.ui.progress(_('scanning'), None)
 
         return parents
 
     def mergesplicemap(self, parents, splicemap):
         """A splicemap redefines child/parent relationships. Check the
         map contains valid revision identifiers and merge the new
         links in the source graph.
         """
         for c in sorted(splicemap):
             if c not in parents:
                 if not self.dest.hascommitforsplicemap(self.map.get(c, c)):
                     # Could be in source but not converted during this run
                     self.ui.warn(_('splice map revision %s is not being '
                                    'converted, ignoring\n') % c)
                 continue
             pc = []
             for p in splicemap[c]:
                 # We do not have to wait for nodes already in dest.
                 if self.dest.hascommitforsplicemap(self.map.get(p, p)):
                     continue
                 # Parent is not in dest and not being converted, not good
                 if p not in parents:
                     raise error.Abort(_('unknown splice map parent: %s') % p)
                 pc.append(p)
             parents[c] = pc
 
     def toposort(self, parents, sortmode):
         '''Return an ordering such that every uncommitted changeset is
         preceded by all its uncommitted ancestors.'''
 
         def mapchildren(parents):
             """Return a (children, roots) tuple where 'children' maps parent
             revision identifiers to children ones, and 'roots' is the list of
             revisions without parents. 'parents' must be a mapping of revision
             identifier to its parents ones.
             """
             visit = sorted(parents)
             seen = set()
             children = {}
             roots = []
 
             while visit:
                 n = visit.pop(0)
                 if n in seen:
                     continue
                 seen.add(n)
                 # Ensure that nodes without parents are present in the
                 # 'children' mapping.
                 children.setdefault(n, [])
                 hasparent = False
                 for p in parents[n]:
                     if p not in self.map:
                         visit.append(p)
                         hasparent = True
                     children.setdefault(p, []).append(n)
                 if not hasparent:
                     roots.append(n)
 
             return children, roots
 
         # Sort functions are supposed to take a list of revisions which
         # can be converted immediately and pick one
 
         def makebranchsorter():
             """If the previously converted revision has a child in the
             eligible revisions list, pick it. Return the list head
             otherwise. Branch sort attempts to minimize branch
             switching, which is harmful for Mercurial backend
             compression.
             """
             prev = [None]
             def picknext(nodes):
                 next = nodes[0]
                 for n in nodes:
                     if prev[0] in parents[n]:
                         next = n
                         break
                 prev[0] = next
                 return next
             return picknext
 
         def makesourcesorter():
             """Source specific sort."""
             keyfn = lambda n: self.commitcache[n].sortkey
             def picknext(nodes):
                 return sorted(nodes, key=keyfn)[0]
             return picknext
 
         def makeclosesorter():
             """Close order sort."""
             keyfn = lambda n: ('close' not in self.commitcache[n].extra,
                                self.commitcache[n].sortkey)
             def picknext(nodes):
                 return sorted(nodes, key=keyfn)[0]
             return picknext
 
         def makedatesorter():
             """Sort revisions by date."""
             dates = {}
             def getdate(n):
                 if n not in dates:
                     dates[n] = util.parsedate(self.commitcache[n].date)
                 return dates[n]
 
             def picknext(nodes):
                 return min([(getdate(n), n) for n in nodes])[1]
 
             return picknext
 
         if sortmode == 'branchsort':
             picknext = makebranchsorter()
         elif sortmode == 'datesort':
             picknext = makedatesorter()
         elif sortmode == 'sourcesort':
             picknext = makesourcesorter()
         elif sortmode == 'closesort':
             picknext = makeclosesorter()
         else:
             raise error.Abort(_('unknown sort mode: %s') % sortmode)
 
         children, actives = mapchildren(parents)
 
         s = []
         pendings = {}
         while actives:
             n = picknext(actives)
             actives.remove(n)
             s.append(n)
 
             # Update dependents list
             for c in children.get(n, []):
                 if c not in pendings:
                     pendings[c] = [p for p in parents[c] if p not in self.map]
                 try:
                     pendings[c].remove(n)
                 except ValueError:
                     raise error.Abort(_('cycle detected between %s and %s')
                                        % (recode(c), recode(n)))
                 if not pendings[c]:
                     # Parents are converted, node is eligible
                     actives.insert(0, c)
                     pendings[c] = None
 
         if len(s) != len(parents):
             raise error.Abort(_("not all revisions were sorted"))
 
         return s
 
     def writeauthormap(self):
         authorfile = self.authorfile
         if authorfile:
             self.ui.status(_('writing author map file %s\n') % authorfile)
             ofile = open(authorfile, 'w+')
             for author in self.authors:
                 ofile.write("%s=%s\n" % (author, self.authors[author]))
             ofile.close()
 
     def readauthormap(self, authorfile):
         afile = open(authorfile, 'r')
         for line in afile:
 
             line = line.strip()
             if not line or line.startswith('#'):
                 continue
 
             try:
                 srcauthor, dstauthor = line.split('=', 1)
             except ValueError:
                 msg = _('ignoring bad line in author map file %s: %s\n')
                 self.ui.warn(msg % (authorfile, line.rstrip()))
                 continue
 
             srcauthor = srcauthor.strip()
             dstauthor = dstauthor.strip()
             if self.authors.get(srcauthor) in (None, dstauthor):
                 msg = _('mapping author %s to %s\n')
                 self.ui.debug(msg % (srcauthor, dstauthor))
                 self.authors[srcauthor] = dstauthor
                 continue
 
             m = _('overriding mapping for author %s, was %s, will be %s\n')
             self.ui.status(m % (srcauthor, self.authors[srcauthor], dstauthor))
 
         afile.close()
 
     def cachecommit(self, rev):
         commit = self.source.getcommit(rev)
         commit.author = self.authors.get(commit.author, commit.author)
         commit.branch = mapbranch(commit.branch, self.branchmap)
         self.commitcache[rev] = commit
         return commit
 
     def copy(self, rev):
         commit = self.commitcache[rev]
         full = self.opts.get('full')
         changes = self.source.getchanges(rev, full)
         if isinstance(changes, basestring):
             if changes == SKIPREV:
                 dest = SKIPREV
             else:
                 dest = self.map[changes]
             self.map[rev] = dest
             return
         files, copies, cleanp2 = changes
         pbranches = []
         if commit.parents:
             for prev in commit.parents:
                 if prev not in self.commitcache:
                     self.cachecommit(prev)
                 pbranches.append((self.map[prev],
                                   self.commitcache[prev].branch))
         self.dest.setbranch(commit.branch, pbranches)
         try:
             parents = self.splicemap[rev]
             self.ui.status(_('spliced in %s as parents of %s\n') %
                            (_(' and ').join(parents), rev))
             parents = [self.map.get(p, p) for p in parents]
         except KeyError:
             parents = [b[0] for b in pbranches]
             parents.extend(self.map[x]
                            for x in commit.optparents
                            if x in self.map)
         if len(pbranches) != 2:
             cleanp2 = set()
         if len(parents) < 3:
             source = progresssource(self.ui, self.source, len(files))
         else:
             # For an octopus merge, we end up traversing the list of
             # changed files N-1 times. This tweak to the number of
             # files makes it so the progress bar doesn't overflow
             # itself.
             source = progresssource(self.ui, self.source,
                                     len(files) * (len(parents) - 1))
         newnode = self.dest.putcommit(files, copies, parents, commit,
                                       source, self.map, full, cleanp2)
         source.close()
         self.source.converted(rev, newnode)
         self.map[rev] = newnode
 
     def convert(self, sortmode):
         try:
             self.source.before()
             self.dest.before()
             self.source.setrevmap(self.map)
             self.ui.status(_("scanning source...\n"))
             heads = self.source.getheads()
             parents = self.walktree(heads)
             self.mergesplicemap(parents, self.splicemap)
             self.ui.status(_("sorting...\n"))
             t = self.toposort(parents, sortmode)
             num = len(t)
             c = None
 
             self.ui.status(_("converting...\n"))
             for i, c in enumerate(t):
                 num -= 1
                 desc = self.commitcache[c].desc
                 if "\n" in desc:
                     desc = desc.splitlines()[0]
                 # convert log message to local encoding without using
                 # tolocal() because the encoding.encoding convert()
                 # uses is 'utf-8'
                 self.ui.status("%d %s\n" % (num, recode(desc)))
                 self.ui.note(_("source: %s\n") % recode(c))
                 self.ui.progress(_('converting'), i, unit=_('revisions'),
                                  total=len(t))
                 self.copy(c)
             self.ui.progress(_('converting'), None)
 
             if not self.ui.configbool('convert', 'skiptags'):
                 tags = self.source.gettags()
                 ctags = {}
                 for k in tags:
                     v = tags[k]
                     if self.map.get(v, SKIPREV) != SKIPREV:
                         ctags[k] = self.map[v]
 
                 if c and ctags:
                     nrev, tagsparent = self.dest.puttags(ctags)
                     if nrev and tagsparent:
                         # write another hash correspondence to override the
                         # previous one so we don't end up with extra tag heads
                         tagsparents = [e for e in self.map.iteritems()
                                        if e[1] == tagsparent]
                         if tagsparents:
                             self.map[tagsparents[0][0]] = nrev
 
             bookmarks = self.source.getbookmarks()
             cbookmarks = {}
             for k in bookmarks:
                 v = bookmarks[k]
                 if self.map.get(v, SKIPREV) != SKIPREV:
                     cbookmarks[k] = self.map[v]
 
             if c and cbookmarks:
                 self.dest.putbookmarks(cbookmarks)
 
             self.writeauthormap()
         finally:
             self.cleanup()
 
     def cleanup(self):
         try:
             self.dest.after()
         finally:
             self.source.after()
         self.map.close()
 
 def convert(ui, src, dest=None, revmapfile=None, **opts):
     global orig_encoding
     orig_encoding = encoding.encoding
     encoding.encoding = 'UTF-8'
 
     # support --authors as an alias for --authormap
     if not opts.get('authormap'):
         opts['authormap'] = opts.get('authors')
 
     if not dest:
         dest = hg.defaultdest(src) + "-hg"
         ui.status(_("assuming destination %s\n") % dest)
 
     destc = convertsink(ui, dest, opts.get('dest_type'))
 
     try:
         srcc, defaultsort = convertsource(ui, src, opts.get('source_type'),
                                           opts.get('rev'))
     except Exception:
         for path in destc.created:
             shutil.rmtree(path, True)
         raise
 
     sortmodes = ('branchsort', 'datesort', 'sourcesort', 'closesort')
     sortmode = [m for m in sortmodes if opts.get(m)]
     if len(sortmode) > 1:
         raise error.Abort(_('more than one sort mode specified'))
     if sortmode:
         sortmode = sortmode[0]
     else:
         sortmode = defaultsort
 
     if sortmode == 'sourcesort' and not srcc.hasnativeorder():
         raise error.Abort(_('--sourcesort is not supported by this data source')
                          )
     if sortmode == 'closesort' and not srcc.hasnativeclose():
         raise error.Abort(_('--closesort is not supported by this data source'))
 
     fmap = opts.get('filemap')
     if fmap:
         srcc = filemap.filemap_source(ui, srcc, fmap)
         destc.setfilemapmode(True)
 
     if not revmapfile:
         revmapfile = destc.revmapfile()
 
     c = converter(ui, srcc, destc, revmapfile, opts)
     c.convert(sortmode)

hgext/convert/cvs.py

@@ -1,296 +1,296
 # cvs.py: CVS conversion code inspired by hg-cvs-import and git-cvsimport
 #
 #  Copyright 2005-2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import errno
 import os
 import re
 import socket
 
+from mercurial.i18n import _
 from mercurial import (
     encoding,
     error,
     util,
 )
-from mercurial.i18n import _
 
 from . import (
     common,
     cvsps,
 )
 
 stringio = util.stringio
 checktool = common.checktool
 commit = common.commit
 converter_source = common.converter_source
 makedatetimestamp = common.makedatetimestamp
 NoRepo = common.NoRepo
 
 class convert_cvs(converter_source):
     def __init__(self, ui, path, revs=None):
         super(convert_cvs, self).__init__(ui, path, revs=revs)
 
         cvs = os.path.join(path, "CVS")
         if not os.path.exists(cvs):
             raise NoRepo(_("%s does not look like a CVS checkout") % path)
 
         checktool('cvs')
 
         self.changeset = None
         self.files = {}
         self.tags = {}
         self.lastbranch = {}
         self.socket = None
         self.cvsroot = open(os.path.join(cvs, "Root")).read()[:-1]
         self.cvsrepo = open(os.path.join(cvs, "Repository")).read()[:-1]
         self.encoding = encoding.encoding
 
         self._connect()
 
     def _parse(self):
         if self.changeset is not None:
             return
         self.changeset = {}
 
         maxrev = 0
         if self.revs:
             if len(self.revs) > 1:
                 raise error.Abort(_('cvs source does not support specifying '
                                    'multiple revs'))
             # TODO: handle tags
             try:
                 # patchset number?
                 maxrev = int(self.revs[0])
             except ValueError:
                 raise error.Abort(_('revision %s is not a patchset number')
                                  % self.revs[0])
 
         d = os.getcwd()
         try:
             os.chdir(self.path)
             id = None
 
             cache = 'update'
             if not self.ui.configbool('convert', 'cvsps.cache', True):
                 cache = None
             db = cvsps.createlog(self.ui, cache=cache)
             db = cvsps.createchangeset(self.ui, db,
                 fuzz=int(self.ui.config('convert', 'cvsps.fuzz', 60)),
                 mergeto=self.ui.config('convert', 'cvsps.mergeto', None),
                 mergefrom=self.ui.config('convert', 'cvsps.mergefrom', None))
 
             for cs in db:
                 if maxrev and cs.id > maxrev:
                     break
                 id = str(cs.id)
                 cs.author = self.recode(cs.author)
                 self.lastbranch[cs.branch] = id
                 cs.comment = self.recode(cs.comment)
                 if self.ui.configbool('convert', 'localtimezone'):
                     cs.date = makedatetimestamp(cs.date[0])
                 date = util.datestr(cs.date, '%Y-%m-%d %H:%M:%S %1%2')
                 self.tags.update(dict.fromkeys(cs.tags, id))
 
                 files = {}
                 for f in cs.entries:
                     files[f.file] = "%s%s" % ('.'.join([str(x)
                                                         for x in f.revision]),
                                               ['', '(DEAD)'][f.dead])
 
                 # add current commit to set
                 c = commit(author=cs.author, date=date,
                            parents=[str(p.id) for p in cs.parents],
                            desc=cs.comment, branch=cs.branch or '')
                 self.changeset[id] = c
                 self.files[id] = files
 
             self.heads = self.lastbranch.values()
         finally:
             os.chdir(d)
 
     def _connect(self):
         root = self.cvsroot
         conntype = None
         user, host = None, None
         cmd = ['cvs', 'server']
 
         self.ui.status(_("connecting to %s\n") % root)
 
         if root.startswith(":pserver:"):
             root = root[9:]
             m = re.match(r'(?:(.*?)(?::(.*?))?@)?([^:\/]*)(?::(\d*))?(.*)',
                          root)
             if m:
                 conntype = "pserver"
                 user, passw, serv, port, root = m.groups()
                 if not user:
                     user = "anonymous"
                 if not port:
                     port = 2401
                 else:
                     port = int(port)
                 format0 = ":pserver:%s@%s:%s" % (user, serv, root)
                 format1 = ":pserver:%s@%s:%d%s" % (user, serv, port, root)
 
                 if not passw:
                     passw = "A"
                     cvspass = os.path.expanduser("~/.cvspass")
                     try:
                         pf = open(cvspass)
                         for line in pf.read().splitlines():
                             part1, part2 = line.split(' ', 1)
                             # /1 :pserver:[e-mail]:2401/cvsroot/foo
                             # Ah<Z
                             if part1 == '/1':
                                 part1, part2 = part2.split(' ', 1)
                                 format = format1
                             # :pserver:[e-mail]:/cvsroot/foo Ah<Z
                             else:
                                 format = format0
                             if part1 == format:
                                 passw = part2
                                 break
                         pf.close()
                     except IOError as inst:
                         if inst.errno != errno.ENOENT:
                             if not getattr(inst, 'filename', None):
                                 inst.filename = cvspass
                             raise
 
                 sck = socket.socket()
                 sck.connect((serv, port))
                 sck.send("\n".join(["BEGIN AUTH REQUEST", root, user, passw,
                                     "END AUTH REQUEST", ""]))
                 if sck.recv(128) != "I LOVE YOU\n":
                     raise error.Abort(_("CVS pserver authentication failed"))
 
                 self.writep = self.readp = sck.makefile('r+')
 
         if not conntype and root.startswith(":local:"):
             conntype = "local"
             root = root[7:]
 
         if not conntype:
             # :ext:user@host/home/user/path/to/cvsroot
             if root.startswith(":ext:"):
                 root = root[5:]
             m = re.match(r'(?:([^@:/]+)@)?([^:/]+):?(.*)', root)
             # Do not take Windows path "c:\foo\bar" for a connection strings
             if os.path.isdir(root) or not m:
                 conntype = "local"
             else:
                 conntype = "rsh"
                 user, host, root = m.group(1), m.group(2), m.group(3)
 
         if conntype != "pserver":
             if conntype == "rsh":
                 rsh = os.environ.get("CVS_RSH") or "ssh"
                 if user:
                     cmd = [rsh, '-l', user, host] + cmd
                 else:
                     cmd = [rsh, host] + cmd
 
             # popen2 does not support argument lists under Windows
             cmd = [util.shellquote(arg) for arg in cmd]
             cmd = util.quotecommand(' '.join(cmd))
             self.writep, self.readp = util.popen2(cmd)
 
         self.realroot = root
 
         self.writep.write("Root %s\n" % root)
         self.writep.write("Valid-responses ok error Valid-requests Mode"
                           " M Mbinary E Checked-in Created Updated"
                           " Merged Removed\n")
         self.writep.write("valid-requests\n")
         self.writep.flush()
         r = self.readp.readline()
         if not r.startswith("Valid-requests"):
             raise error.Abort(_('unexpected response from CVS server '
                                '(expected "Valid-requests", but got %r)')
                              % r)
         if "UseUnchanged" in r:
             self.writep.write("UseUnchanged\n")
             self.writep.flush()
             r = self.readp.readline()
 
     def getheads(self):
         self._parse()
         return self.heads
 
     def getfile(self, name, rev):
 
         def chunkedread(fp, count):
             # file-objects returned by socket.makefile() do not handle
             # large read() requests very well.
             chunksize = 65536
             output = stringio()
             while count > 0:
                 data = fp.read(min(count, chunksize))
                 if not data:
                     raise error.Abort(_("%d bytes missing from remote file")
                                      % count)
                 count -= len(data)
                 output.write(data)
             return output.getvalue()
 
         self._parse()
         if rev.endswith("(DEAD)"):
             return None, None
 
         args = ("-N -P -kk -r %s --" % rev).split()
         args.append(self.cvsrepo + '/' + name)
         for x in args:
             self.writep.write("Argument %s\n" % x)
         self.writep.write("Directory .\n%s\nco\n" % self.realroot)
         self.writep.flush()
 
         data = ""
         mode = None
         while True:
             line = self.readp.readline()
             if line.startswith("Created ") or line.startswith("Updated "):
                 self.readp.readline() # path
                 self.readp.readline() # entries
                 mode = self.readp.readline()[:-1]
                 count = int(self.readp.readline()[:-1])
                 data = chunkedread(self.readp, count)
             elif line.startswith(" "):
                 data += line[1:]
             elif line.startswith("M "):
                 pass
             elif line.startswith("Mbinary "):
                 count = int(self.readp.readline()[:-1])
                 data = chunkedread(self.readp, count)
             else:
                 if line == "ok\n":
                     if mode is None:
                         raise error.Abort(_('malformed response from CVS'))
                     return (data, "x" in mode and "x" or "")
                 elif line.startswith("E "):
                     self.ui.warn(_("cvs server: %s\n") % line[2:])
                 elif line.startswith("Remove"):
                     self.readp.readline()
                 else:
                     raise error.Abort(_("unknown CVS response: %s") % line)
 
     def getchanges(self, rev, full):
         if full:
             raise error.Abort(_("convert from cvs does not support --full"))
         self._parse()
         return sorted(self.files[rev].iteritems()), {}, set()
 
     def getcommit(self, rev):
         self._parse()
         return self.changeset[rev]
 
     def gettags(self):
         self._parse()
         return self.tags
 
     def getchangedfiles(self, rev, i):
         self._parse()
         return sorted(self.files[rev])

hgext/convert/cvsps.py

@@ -1,918 +1,919
 # Mercurial built-in replacement for cvsps.
 #
 # Copyright 2008, Frank Kingswood <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
-import cPickle as pickle
 import os
 import re
 
+from mercurial.i18n import _
 from mercurial import (
     hook,
     util,
 )
-from mercurial.i18n import _
+
+pickle = util.pickle
 
 class logentry(object):
     '''Class logentry has the following attributes:
         .author    - author name as CVS knows it
         .branch    - name of branch this revision is on
         .branches  - revision tuple of branches starting at this revision
         .comment   - commit message
         .commitid  - CVS commitid or None
         .date      - the commit date as a (time, tz) tuple
         .dead      - true if file revision is dead
         .file      - Name of file
         .lines     - a tuple (+lines, -lines) or None
         .parent    - Previous revision of this entry
         .rcs       - name of file as returned from CVS
         .revision  - revision number as tuple
         .tags      - list of tags on the file
         .synthetic - is this a synthetic "file ... added on ..." revision?
         .mergepoint - the branch that has been merged from (if present in
                       rlog output) or None
         .branchpoints - the branches that start at the current entry or empty
     '''
     def __init__(self, **entries):
         self.synthetic = False
         self.__dict__.update(entries)
 
     def __repr__(self):
         items = ("%s=%r"%(k, self.__dict__[k]) for k in sorted(self.__dict__))
         return "%s(%s)"%(type(self).__name__, ", ".join(items))
 
 class logerror(Exception):
     pass
 
 def getrepopath(cvspath):
     """Return the repository path from a CVS path.
 
     >>> getrepopath('/foo/bar')
     '/foo/bar'
     >>> getrepopath('c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(':pserver:10/foo/bar')
     '/foo/bar'
     >>> getrepopath(':pserver:10c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(':pserver:/foo/bar')
     '/foo/bar'
     >>> getrepopath(':pserver:c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(':pserver:[e-mail]:/foo/bar')
     '/foo/bar'
     >>> getrepopath(':pserver:[e-mail]:c:/foo/bar')
     '/foo/bar'
     >>> getrepopath('user@server/path/to/repository')
     '/path/to/repository'
     """
     # According to CVS manual, CVS paths are expressed like:
     # [:method:][[user][:password]@]hostname[:[port]]/path/to/repository
     #
     # CVSpath is splitted into parts and then position of the first occurrence
     # of the '/' char after the '@' is located. The solution is the rest of the
     # string after that '/' sign including it
 
     parts = cvspath.split(':')
     atposition = parts[-1].find('@')
     start = 0
 
     if atposition != -1:
         start = atposition
 
     repopath = parts[-1][parts[-1].find('/', start):]
     return repopath
 
 def createlog(ui, directory=None, root="", rlog=True, cache=None):
     '''Collect the CVS rlog'''
 
     # Because we store many duplicate commit log messages, reusing strings
     # saves a lot of memory and pickle storage space.
     _scache = {}
     def scache(s):
         "return a shared version of a string"
         return _scache.setdefault(s, s)
 
     ui.status(_('collecting CVS rlog\n'))
 
     log = []      # list of logentry objects containing the CVS state
 
     # patterns to match in CVS (r)log output, by state of use
     re_00 = re.compile('RCS file: (.+)$')
     re_01 = re.compile('cvs \\[r?log aborted\\]: (.+)$')
     re_02 = re.compile('cvs (r?log|server): (.+)\n$')
     re_03 = re.compile("(Cannot access.+CVSROOT)|"
                        "(can't create temporary directory.+)$")
     re_10 = re.compile('Working file: (.+)$')
     re_20 = re.compile('symbolic names:')
     re_30 = re.compile('\t(.+): ([\\d.]+)$')
     re_31 = re.compile('----------------------------$')
     re_32 = re.compile('======================================='
                        '======================================$')
     re_50 = re.compile('revision ([\\d.]+)(\s+locked by:\s+.+;)?$')
     re_60 = re.compile(r'date:\s+(.+);\s+author:\s+(.+);\s+state:\s+(.+?);'
                        r'(\s+lines:\s+(\+\d+)?\s+(-\d+)?;)?'
                        r'(\s+commitid:\s+([^;]+);)?'
                        r'(.*mergepoint:\s+([^;]+);)?')
     re_70 = re.compile('branches: (.+);$')
 
     file_added_re = re.compile(r'file [^/]+ was (initially )?added on branch')
 
     prefix = ''   # leading path to strip of what we get from CVS
 
     if directory is None:
         # Current working directory
 
         # Get the real directory in the repository
         try:
             prefix = open(os.path.join('CVS','Repository')).read().strip()
             directory = prefix
             if prefix == ".":
                 prefix = ""
         except IOError:
             raise logerror(_('not a CVS sandbox'))
 
         if prefix and not prefix.endswith(os.sep):
             prefix += os.sep
 
         # Use the Root file in the sandbox, if it exists
         try:
             root = open(os.path.join('CVS','Root')).read().strip()
         except IOError:
             pass
 
     if not root:
         root = os.environ.get('CVSROOT', '')
 
     # read log cache if one exists
     oldlog = []
     date = None
 
     if cache:
         cachedir = os.path.expanduser('~/.hg.cvsps')
         if not os.path.exists(cachedir):
             os.mkdir(cachedir)
 
         # The cvsps cache pickle needs a uniquified name, based on the
         # repository location. The address may have all sort of nasties
         # in it, slashes, colons and such. So here we take just the
         # alphanumeric characters, concatenated in a way that does not
         # mix up the various components, so that
         #    :pserver:user@server:/path
         # and
         #    /pserver/user/server/path
         # are mapped to different cache file names.
         cachefile = root.split(":") + [directory, "cache"]
         cachefile = ['-'.join(re.findall(r'\w+', s)) for s in cachefile if s]
         cachefile = os.path.join(cachedir,
                                  '.'.join([s for s in cachefile if s]))
 
     if cache == 'update':
         try:
             ui.note(_('reading cvs log cache %s\n') % cachefile)
             oldlog = pickle.load(open(cachefile))
             for e in oldlog:
                 if not (util.safehasattr(e, 'branchpoints') and
                         util.safehasattr(e, 'commitid') and
                         util.safehasattr(e, 'mergepoint')):
                     ui.status(_('ignoring old cache\n'))
                     oldlog = []
                     break
 
             ui.note(_('cache has %d log entries\n') % len(oldlog))
         except Exception as e:
             ui.note(_('error reading cache: %r\n') % e)
 
         if oldlog:
             date = oldlog[-1].date    # last commit date as a (time,tz) tuple
             date = util.datestr(date, '%Y/%m/%d %H:%M:%S %1%2')
 
     # build the CVS commandline
     cmd = ['cvs', '-q']
     if root:
         cmd.append('-d%s' % root)
         p = util.normpath(getrepopath(root))
         if not p.endswith('/'):
             p += '/'
         if prefix:
             # looks like normpath replaces "" by "."
             prefix = p + util.normpath(prefix)
         else:
             prefix = p
     cmd.append(['log', 'rlog'][rlog])
     if date:
         # no space between option and date string
         cmd.append('-d>%s' % date)
     cmd.append(directory)
 
     # state machine begins here
     tags = {}     # dictionary of revisions on current file with their tags
     branchmap = {} # mapping between branch names and revision numbers
     rcsmap = {}
     state = 0
     store = False # set when a new record can be appended
 
     cmd = [util.shellquote(arg) for arg in cmd]
     ui.note(_("running %s\n") % (' '.join(cmd)))
     ui.debug("prefix=%r directory=%r root=%r\n" % (prefix, directory, root))
 
     pfp = util.popen(' '.join(cmd))
     peek = pfp.readline()
     while True:
         line = peek
         if line == '':
             break
         peek = pfp.readline()
         if line.endswith('\n'):
             line = line[:-1]
         #ui.debug('state=%d line=%r\n' % (state, line))
 
         if state == 0:
             # initial state, consume input until we see 'RCS file'
             match = re_00.match(line)
             if match:
                 rcs = match.group(1)
                 tags = {}
                 if rlog:
                     filename = util.normpath(rcs[:-2])
                     if filename.startswith(prefix):
                         filename = filename[len(prefix):]
                     if filename.startswith('/'):
                         filename = filename[1:]
                     if filename.startswith('Attic/'):
                         filename = filename[6:]
                     else:
                         filename = filename.replace('/Attic/', '/')
                     state = 2
                     continue
                 state = 1
                 continue
             match = re_01.match(line)
             if match:
                 raise logerror(match.group(1))
             match = re_02.match(line)
             if match:
                 raise logerror(match.group(2))
             if re_03.match(line):
                 raise logerror(line)
 
         elif state == 1:
             # expect 'Working file' (only when using log instead of rlog)
             match = re_10.match(line)
             assert match, _('RCS file must be followed by working file')
             filename = util.normpath(match.group(1))
             state = 2
 
         elif state == 2:
             # expect 'symbolic names'
             if re_20.match(line):
                 branchmap = {}
                 state = 3
 
         elif state == 3:
             # read the symbolic names and store as tags
             match = re_30.match(line)
             if match:
                 rev = [int(x) for x in match.group(2).split('.')]
 
                 # Convert magic branch number to an odd-numbered one
                 revn = len(rev)
                 if revn > 3 and (revn % 2) == 0 and rev[-2] == 0:
                     rev = rev[:-2] + rev[-1:]
                 rev = tuple(rev)
 
                 if rev not in tags:
                     tags[rev] = []
                 tags[rev].append(match.group(1))
                 branchmap[match.group(1)] = match.group(2)
 
             elif re_31.match(line):
                 state = 5
             elif re_32.match(line):
                 state = 0
 
         elif state == 4:
             # expecting '------' separator before first revision
             if re_31.match(line):
                 state = 5
             else:
                 assert not re_32.match(line), _('must have at least '
                                                 'some revisions')
 
         elif state == 5:
             # expecting revision number and possibly (ignored) lock indication
             # we create the logentry here from values stored in states 0 to 4,
             # as this state is re-entered for subsequent revisions of a file.
             match = re_50.match(line)
             assert match, _('expected revision number')
             e = logentry(rcs=scache(rcs),
                          file=scache(filename),
                          revision=tuple([int(x) for x in
                                          match.group(1).split('.')]),
                          branches=[],
                          parent=None,
                          commitid=None,
                          mergepoint=None,
                          branchpoints=set())
 
             state = 6
 
         elif state == 6:
             # expecting date, author, state, lines changed
             match = re_60.match(line)
             assert match, _('revision must be followed by date line')
             d = match.group(1)
             if d[2] == '/':
                 # Y2K
                 d = '19' + d
 
             if len(d.split()) != 3:
                 # cvs log dates always in GMT
                 d = d + ' UTC'
             e.date = util.parsedate(d, ['%y/%m/%d %H:%M:%S',
                                         '%Y/%m/%d %H:%M:%S',
                                         '%Y-%m-%d %H:%M:%S'])
             e.author = scache(match.group(2))
             e.dead = match.group(3).lower() == 'dead'
 
             if match.group(5):
                 if match.group(6):
                     e.lines = (int(match.group(5)), int(match.group(6)))
                 else:
                     e.lines = (int(match.group(5)), 0)
             elif match.group(6):
                 e.lines = (0, int(match.group(6)))
             else:
                 e.lines = None
 
             if match.group(7): # cvs 1.12 commitid
                 e.commitid = match.group(8)
 
             if match.group(9): # cvsnt mergepoint
                 myrev = match.group(10).split('.')
                 if len(myrev) == 2: # head
                     e.mergepoint = 'HEAD'
                 else:
                     myrev = '.'.join(myrev[:-2] + ['0', myrev[-2]])
                     branches = [b for b in branchmap if branchmap[b] == myrev]
                     assert len(branches) == 1, ('unknown branch: %s'
                                                 % e.mergepoint)
                     e.mergepoint = branches[0]
 
             e.comment = []
             state = 7
 
         elif state == 7:
             # read the revision numbers of branches that start at this revision
             # or store the commit log message otherwise
             m = re_70.match(line)
             if m:
                 e.branches = [tuple([int(y) for y in x.strip().split('.')])
                                 for x in m.group(1).split(';')]
                 state = 8
             elif re_31.match(line) and re_50.match(peek):
                 state = 5
                 store = True
             elif re_32.match(line):
                 state = 0
                 store = True
             else:
                 e.comment.append(line)
 
         elif state == 8:
             # store commit log message
             if re_31.match(line):
                 cpeek = peek
                 if cpeek.endswith('\n'):
                     cpeek = cpeek[:-1]
                 if re_50.match(cpeek):
                     state = 5
                     store = True
                 else:
                     e.comment.append(line)
             elif re_32.match(line):
                 state = 0
                 store = True
             else:
                 e.comment.append(line)
 
         # When a file is added on a branch B1, CVS creates a synthetic
         # dead trunk revision 1.1 so that the branch has a root.
         # Likewise, if you merge such a file to a later branch B2 (one
         # that already existed when the file was added on B1), CVS
         # creates a synthetic dead revision 1.1.x.1 on B2.  Don't drop
         # these revisions now, but mark them synthetic so
         # createchangeset() can take care of them.
         if (store and
               e.dead and
               e.revision[-1] == 1 and      # 1.1 or 1.1.x.1
               len(e.comment) == 1 and
               file_added_re.match(e.comment[0])):
             ui.debug('found synthetic revision in %s: %r\n'
                      % (e.rcs, e.comment[0]))
             e.synthetic = True
 
         if store:
             # clean up the results and save in the log.
             store = False
             e.tags = sorted([scache(x) for x in tags.get(e.revision, [])])
             e.comment = scache('\n'.join(e.comment))
 
             revn = len(e.revision)
             if revn > 3 and (revn % 2) == 0:
                 e.branch = tags.get(e.revision[:-1], [None])[0]
             else:
                 e.branch = None
 
             # find the branches starting from this revision
             branchpoints = set()
             for branch, revision in branchmap.iteritems():
                 revparts = tuple([int(i) for i in revision.split('.')])
                 if len(revparts) < 2: # bad tags
                     continue
                 if revparts[-2] == 0 and revparts[-1] % 2 == 0:
                     # normal branch
                     if revparts[:-2] == e.revision:
                         branchpoints.add(branch)
                 elif revparts == (1, 1, 1): # vendor branch
                     if revparts in e.branches:
                         branchpoints.add(branch)
             e.branchpoints = branchpoints
 
             log.append(e)
 
             rcsmap[e.rcs.replace('/Attic/', '/')] = e.rcs
 
             if len(log) % 100 == 0:
                 ui.status(util.ellipsis('%d %s' % (len(log), e.file), 80)+'\n')
 
     log.sort(key=lambda x: (x.rcs, x.revision))
 
     # find parent revisions of individual files
     versions = {}
     for e in sorted(oldlog, key=lambda x: (x.rcs, x.revision)):
         rcs = e.rcs.replace('/Attic/', '/')
         if rcs in rcsmap:
             e.rcs = rcsmap[rcs]
         branch = e.revision[:-1]
         versions[(e.rcs, branch)] = e.revision
 
     for e in log:
         branch = e.revision[:-1]
         p = versions.get((e.rcs, branch), None)
         if p is None:
             p = e.revision[:-2]
         e.parent = p
         versions[(e.rcs, branch)] = e.revision
 
     # update the log cache
     if cache:
         if log:
             # join up the old and new logs
             log.sort(key=lambda x: x.date)
 
             if oldlog and oldlog[-1].date >= log[0].date:
                 raise logerror(_('log cache overlaps with new log entries,'
                                  ' re-run without cache.'))
 
             log = oldlog + log
 
             # write the new cachefile
             ui.note(_('writing cvs log cache %s\n') % cachefile)
             pickle.dump(log, open(cachefile, 'w'))
         else:
             log = oldlog
 
     ui.status(_('%d log entries\n') % len(log))
 
     hook.hook(ui, None, "cvslog", True, log=log)
 
     return log
 
 
 class changeset(object):
     '''Class changeset has the following attributes:
         .id        - integer identifying this changeset (list index)
         .author    - author name as CVS knows it
         .branch    - name of branch this changeset is on, or None
         .comment   - commit message
         .commitid  - CVS commitid or None
         .date      - the commit date as a (time,tz) tuple
         .entries   - list of logentry objects in this changeset
         .parents   - list of one or two parent changesets
         .tags      - list of tags on this changeset
         .synthetic - from synthetic revision "file ... added on branch ..."
         .mergepoint- the branch that has been merged from or None
         .branchpoints- the branches that start at the current entry or empty
     '''
     def __init__(self, **entries):
         self.id = None
         self.synthetic = False
         self.__dict__.update(entries)
 
     def __repr__(self):
         items = ("%s=%r"%(k, self.__dict__[k]) for k in sorted(self.__dict__))
         return "%s(%s)"%(type(self).__name__, ", ".join(items))
 
 def createchangeset(ui, log, fuzz=60, mergefrom=None, mergeto=None):
     '''Convert log into changesets.'''
 
     ui.status(_('creating changesets\n'))
 
     # try to order commitids by date
     mindate = {}
     for e in log:
         if e.commitid:
             mindate[e.commitid] = min(e.date, mindate.get(e.commitid))
 
     # Merge changesets
     log.sort(key=lambda x: (mindate.get(x.commitid), x.commitid, x.comment,
                             x.author, x.branch, x.date, x.branchpoints))
 
     changesets = []
     files = set()
     c = None
     for i, e in enumerate(log):
 
         # Check if log entry belongs to the current changeset or not.
 
         # Since CVS is file-centric, two different file revisions with
         # different branchpoints should be treated as belonging to two
         # different changesets (and the ordering is important and not
         # honoured by cvsps at this point).
         #
         # Consider the following case:
         # foo 1.1 branchpoints: [MYBRANCH]
         # bar 1.1 branchpoints: [MYBRANCH, MYBRANCH2]
         #
         # Here foo is part only of MYBRANCH, but not MYBRANCH2, e.g. a
         # later version of foo may be in MYBRANCH2, so foo should be the
         # first changeset and bar the next and MYBRANCH and MYBRANCH2
         # should both start off of the bar changeset. No provisions are
         # made to ensure that this is, in fact, what happens.
         if not (c and e.branchpoints == c.branchpoints and
                 (# cvs commitids
                  (e.commitid is not None and e.commitid == c.commitid) or
                  (# no commitids, use fuzzy commit detection
                   (e.commitid is None or c.commitid is None) and
                    e.comment == c.comment and
                    e.author == c.author and
                    e.branch == c.branch and
                    ((c.date[0] + c.date[1]) <=
                     (e.date[0] + e.date[1]) <=
                     (c.date[0] + c.date[1]) + fuzz) and
                    e.file not in files))):
             c = changeset(comment=e.comment, author=e.author,
                           branch=e.branch, date=e.date,
                           entries=[], mergepoint=e.mergepoint,
                           branchpoints=e.branchpoints, commitid=e.commitid)
             changesets.append(c)
 
             files = set()
             if len(changesets) % 100 == 0:
                 t = '%d %s' % (len(changesets), repr(e.comment)[1:-1])
                 ui.status(util.ellipsis(t, 80) + '\n')
 
         c.entries.append(e)
         files.add(e.file)
         c.date = e.date       # changeset date is date of latest commit in it
 
     # Mark synthetic changesets
 
     for c in changesets:
         # Synthetic revisions always get their own changeset, because
         # the log message includes the filename.  E.g. if you add file3
         # and file4 on a branch, you get four log entries and three
         # changesets:
         #   "File file3 was added on branch ..." (synthetic, 1 entry)
         #   "File file4 was added on branch ..." (synthetic, 1 entry)
         #   "Add file3 and file4 to fix ..."     (real, 2 entries)
         # Hence the check for 1 entry here.
         c.synthetic = len(c.entries) == 1 and c.entries[0].synthetic
 
     # Sort files in each changeset
 
     def entitycompare(l, r):
         'Mimic cvsps sorting order'
         l = l.file.split('/')
         r = r.file.split('/')
         nl = len(l)
         nr = len(r)
         n = min(nl, nr)
         for i in range(n):
             if i + 1 == nl and nl < nr:
                 return -1
             elif i + 1 == nr and nl > nr:
                 return +1
             elif l[i] < r[i]:
                 return -1
             elif l[i] > r[i]:
                 return +1
         return 0
 
     for c in changesets:
         c.entries.sort(entitycompare)
 
     # Sort changesets by date
 
     odd = set()
     def cscmp(l, r, odd=odd):
         d = sum(l.date) - sum(r.date)
         if d:
             return d
 
         # detect vendor branches and initial commits on a branch
         le = {}
         for e in l.entries:
             le[e.rcs] = e.revision
         re = {}
         for e in r.entries:
             re[e.rcs] = e.revision
 
         d = 0
         for e in l.entries:
             if re.get(e.rcs, None) == e.parent:
                 assert not d
                 d = 1
                 break
 
         for e in r.entries:
             if le.get(e.rcs, None) == e.parent:
                 if d:
                     odd.add((l, r))
                 d = -1
                 break
         # By this point, the changesets are sufficiently compared that
         # we don't really care about ordering. However, this leaves
         # some race conditions in the tests, so we compare on the
         # number of files modified, the files contained in each
         # changeset, and the branchpoints in the change to ensure test
         # output remains stable.
 
         # recommended replacement for cmp from
         # https://docs.python.org/3.0/whatsnew/3.0.html
         c = lambda x, y: (x > y) - (x < y)
         # Sort bigger changes first.
         if not d:
             d = c(len(l.entries), len(r.entries))
         # Try sorting by filename in the change.
         if not d:
             d = c([e.file for e in l.entries], [e.file for e in r.entries])
         # Try and put changes without a branch point before ones with
         # a branch point.
         if not d:
             d = c(len(l.branchpoints), len(r.branchpoints))
         return d
 
     changesets.sort(cscmp)
 
     # Collect tags
 
     globaltags = {}
     for c in changesets:
         for e in c.entries:
             for tag in e.tags:
                 # remember which is the latest changeset to have this tag
                 globaltags[tag] = c
 
     for c in changesets:
         tags = set()
         for e in c.entries:
             tags.update(e.tags)
         # remember tags only if this is the latest changeset to have it
         c.tags = sorted(tag for tag in tags if globaltags[tag] is c)
 
     # Find parent changesets, handle {{mergetobranch BRANCHNAME}}
     # by inserting dummy changesets with two parents, and handle
     # {{mergefrombranch BRANCHNAME}} by setting two parents.
 
     if mergeto is None:
         mergeto = r'{{mergetobranch ([-\w]+)}}'
     if mergeto:
         mergeto = re.compile(mergeto)
 
     if mergefrom is None:
         mergefrom = r'{{mergefrombranch ([-\w]+)}}'
     if mergefrom:
         mergefrom = re.compile(mergefrom)
 
     versions = {}    # changeset index where we saw any particular file version
     branches = {}    # changeset index where we saw a branch
     n = len(changesets)
     i = 0
     while i < n:
         c = changesets[i]
 
         for f in c.entries:
             versions[(f.rcs, f.revision)] = i
 
         p = None
         if c.branch in branches:
             p = branches[c.branch]
         else:
             # first changeset on a new branch
             # the parent is a changeset with the branch in its
             # branchpoints such that it is the latest possible
             # commit without any intervening, unrelated commits.
 
             for candidate in xrange(i):
                 if c.branch not in changesets[candidate].branchpoints:
                     if p is not None:
                         break
                     continue
                 p = candidate
 
         c.parents = []
         if p is not None:
             p = changesets[p]
 
             # Ensure no changeset has a synthetic changeset as a parent.
             while p.synthetic:
                 assert len(p.parents) <= 1, \
                        _('synthetic changeset cannot have multiple parents')
                 if p.parents:
                     p = p.parents[0]
                 else:
                     p = None
                     break
 
             if p is not None:
                 c.parents.append(p)
 
         if c.mergepoint:
             if c.mergepoint == 'HEAD':
                 c.mergepoint = None
             c.parents.append(changesets[branches[c.mergepoint]])
 
         if mergefrom:
             m = mergefrom.search(c.comment)
             if m:
                 m = m.group(1)
                 if m == 'HEAD':
                     m = None
                 try:
                     candidate = changesets[branches[m]]
                 except KeyError:
                     ui.warn(_("warning: CVS commit message references "
                               "non-existent branch %r:\n%s\n")
                             % (m, c.comment))
                 if m in branches and c.branch != m and not candidate.synthetic:
                     c.parents.append(candidate)
 
         if mergeto:
             m = mergeto.search(c.comment)
             if m:
                 if m.groups():
                     m = m.group(1)
                     if m == 'HEAD':
                         m = None
                 else:
                     m = None   # if no group found then merge to HEAD
                 if m in branches and c.branch != m:
                     # insert empty changeset for merge
                     cc = changeset(
                         author=c.author, branch=m, date=c.date,
                         comment='convert-repo: CVS merge from branch %s'
                         % c.branch,
                         entries=[], tags=[],
                         parents=[changesets[branches[m]], c])
                     changesets.insert(i + 1, cc)
                     branches[m] = i + 1
 
                     # adjust our loop counters now we have inserted a new entry
                     n += 1
                     i += 2
                     continue
 
         branches[c.branch] = i
         i += 1
 
     # Drop synthetic changesets (safe now that we have ensured no other
     # changesets can have them as parents).
     i = 0
     while i < len(changesets):
         if changesets[i].synthetic:
             del changesets[i]
         else:
             i += 1
 
     # Number changesets
 
     for i, c in enumerate(changesets):
         c.id = i + 1
 
     if odd:
         for l, r in odd:
             if l.id is not None and r.id is not None:
                 ui.warn(_('changeset %d is both before and after %d\n')
                         % (l.id, r.id))
 
     ui.status(_('%d changeset entries\n') % len(changesets))
 
     hook.hook(ui, None, "cvschangesets", True, changesets=changesets)
 
     return changesets
 
 
 def debugcvsps(ui, *args, **opts):
     '''Read CVS rlog for current directory or named path in
     repository, and convert the log to changesets based on matching
     commit log entries and dates.
     '''
     if opts["new_cache"]:
         cache = "write"
     elif opts["update_cache"]:
         cache = "update"
     else:
         cache = None
 
     revisions = opts["revisions"]
 
     try:
         if args:
             log = []
             for d in args:
                 log += createlog(ui, d, root=opts["root"], cache=cache)
         else:
             log = createlog(ui, root=opts["root"], cache=cache)
     except logerror as e:
         ui.write("%r\n"%e)
         return
 
     changesets = createchangeset(ui, log, opts["fuzz"])
     del log
 
     # Print changesets (optionally filtered)
 
     off = len(revisions)
     branches = {}    # latest version number in each branch
     ancestors = {}   # parent branch
     for cs in changesets:
 
         if opts["ancestors"]:
             if cs.branch not in branches and cs.parents and cs.parents[0].id:
                 ancestors[cs.branch] = (changesets[cs.parents[0].id - 1].branch,
                                         cs.parents[0].id)
             branches[cs.branch] = cs.id
 
         # limit by branches
         if opts["branches"] and (cs.branch or 'HEAD') not in opts["branches"]:
             continue
 
         if not off:
             # Note: trailing spaces on several lines here are needed to have
             #       bug-for-bug compatibility with cvsps.
             ui.write('---------------------\n')
             ui.write(('PatchSet %d \n' % cs.id))
             ui.write(('Date: %s\n' % util.datestr(cs.date,
                                                  '%Y/%m/%d %H:%M:%S %1%2')))
             ui.write(('Author: %s\n' % cs.author))
             ui.write(('Branch: %s\n' % (cs.branch or 'HEAD')))
             ui.write(('Tag%s: %s \n' % (['', 's'][len(cs.tags) > 1],
                                   ','.join(cs.tags) or '(none)')))
             if cs.branchpoints:
                 ui.write(('Branchpoints: %s \n') %
                          ', '.join(sorted(cs.branchpoints)))
             if opts["parents"] and cs.parents:
                 if len(cs.parents) > 1:
                     ui.write(('Parents: %s\n' %
                              (','.join([str(p.id) for p in cs.parents]))))
                 else:
                     ui.write(('Parent: %d\n' % cs.parents[0].id))
 
             if opts["ancestors"]:
                 b = cs.branch
                 r = []
                 while b:
                     b, c = ancestors[b]
                     r.append('%s:%d:%d' % (b or "HEAD", c, branches[b]))
                 if r:
                     ui.write(('Ancestors: %s\n' % (','.join(r))))
 
             ui.write(('Log:\n'))
             ui.write('%s\n\n' % cs.comment)
             ui.write(('Members: \n'))
             for f in cs.entries:
                 fn = f.file
                 if fn.startswith(opts["prefix"]):
                     fn = fn[len(opts["prefix"]):]
                 ui.write('\t%s:%s->%s%s \n' % (
                         fn, '.'.join([str(x) for x in f.parent]) or 'INITIAL',
                         '.'.join([str(x) for x in f.revision]),
                         ['', '(DEAD)'][f.dead]))
             ui.write('\n')
 
         # have we seen the start tag?
         if revisions and off:
             if revisions[0] == str(cs.id) or \
                 revisions[0] in cs.tags:
                 off = False
 
         # see if we reached the end tag
         if len(revisions) > 1 and not off:
             if revisions[1] == str(cs.id) or \
                 revisions[1] in cs.tags:
                 break

hgext/convert/filemap.py

@@ -1,462 +1,463
 # Copyright 2007 Bryan O'Sullivan <[e-mail]>
 # Copyright 2007 Alexis S. L. Carvalho <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import posixpath
 import shlex
+
+from mercurial.i18n import _
 from mercurial import (
     error,
 )
-from mercurial.i18n import _
 from . import common
 SKIPREV = common.SKIPREV
 
 def rpairs(path):
     '''Yield tuples with path split at '/', starting with the full path.
     No leading, trailing or double '/', please.
     >>> for x in rpairs('foo/bar/baz'): print x
     ('foo/bar/baz', '')
     ('foo/bar', 'baz')
     ('foo', 'bar/baz')
     ('.', 'foo/bar/baz')
     '''
     i = len(path)
     while i != -1:
         yield path[:i], path[i + 1:]
         i = path.rfind('/', 0, i)
     yield '.', path
 
 def normalize(path):
     ''' We use posixpath.normpath to support cross-platform path format.
     However, it doesn't handle None input. So we wrap it up. '''
     if path is None:
         return None
     return posixpath.normpath(path)
 
 class filemapper(object):
     '''Map and filter filenames when importing.
     A name can be mapped to itself, a new name, or None (omit from new
     repository).'''
 
     def __init__(self, ui, path=None):
         self.ui = ui
         self.include = {}
         self.exclude = {}
         self.rename = {}
         self.targetprefixes = None
         if path:
             if self.parse(path):
                 raise error.Abort(_('errors in filemap'))
 
     def parse(self, path):
         errs = 0
         def check(name, mapping, listname):
             if not name:
                 self.ui.warn(_('%s:%d: path to %s is missing\n') %
                              (lex.infile, lex.lineno, listname))
                 return 1
             if name in mapping:
                 self.ui.warn(_('%s:%d: %r already in %s list\n') %
                              (lex.infile, lex.lineno, name, listname))
                 return 1
             if (name.startswith('/') or
                 name.endswith('/') or
                 '//' in name):
                 self.ui.warn(_('%s:%d: superfluous / in %s %r\n') %
                              (lex.infile, lex.lineno, listname, name))
                 return 1
             return 0
         lex = shlex.shlex(open(path), path, True)
         lex.wordchars += '!@#$%^&*()-=+[]{}|;:,./<>?'
         cmd = lex.get_token()
         while cmd:
             if cmd == 'include':
                 name = normalize(lex.get_token())
                 errs += check(name, self.exclude, 'exclude')
                 self.include[name] = name
             elif cmd == 'exclude':
                 name = normalize(lex.get_token())
                 errs += check(name, self.include, 'include')
                 errs += check(name, self.rename, 'rename')
                 self.exclude[name] = name
             elif cmd == 'rename':
                 src = normalize(lex.get_token())
                 dest = normalize(lex.get_token())
                 errs += check(src, self.exclude, 'exclude')
                 self.rename[src] = dest
             elif cmd == 'source':
                 errs += self.parse(normalize(lex.get_token()))
             else:
                 self.ui.warn(_('%s:%d: unknown directive %r\n') %
                              (lex.infile, lex.lineno, cmd))
                 errs += 1
             cmd = lex.get_token()
         return errs
 
     def lookup(self, name, mapping):
         name = normalize(name)
         for pre, suf in rpairs(name):
             try:
                 return mapping[pre], pre, suf
             except KeyError:
                 pass
         return '', name, ''
 
     def istargetfile(self, filename):
         """Return true if the given target filename is covered as a destination
         of the filemap. This is useful for identifying what parts of the target
         repo belong to the source repo and what parts don't."""
         if self.targetprefixes is None:
             self.targetprefixes = set()
             for before, after in self.rename.iteritems():
                 self.targetprefixes.add(after)
 
         # If "." is a target, then all target files are considered from the
         # source.
         if not self.targetprefixes or '.' in self.targetprefixes:
             return True
 
         filename = normalize(filename)
         for pre, suf in rpairs(filename):
             # This check is imperfect since it doesn't account for the
             # include/exclude list, but it should work in filemaps that don't
             # apply include/exclude to the same source directories they are
             # renaming.
             if pre in self.targetprefixes:
                 return True
         return False
 
     def __call__(self, name):
         if self.include:
             inc = self.lookup(name, self.include)[0]
         else:
             inc = name
         if self.exclude:
             exc = self.lookup(name, self.exclude)[0]
         else:
             exc = ''
         if (not self.include and exc) or (len(inc) <= len(exc)):
             return None
         newpre, pre, suf = self.lookup(name, self.rename)
         if newpre:
             if newpre == '.':
                 return suf
             if suf:
                 if newpre.endswith('/'):
                     return newpre + suf
                 return newpre + '/' + suf
             return newpre
         return name
 
     def active(self):
         return bool(self.include or self.exclude or self.rename)
 
 # This class does two additional things compared to a regular source:
 #
 # - Filter and rename files.  This is mostly wrapped by the filemapper
 #   class above. We hide the original filename in the revision that is
 #   returned by getchanges to be able to find things later in getfile.
 #
 # - Return only revisions that matter for the files we're interested in.
 #   This involves rewriting the parents of the original revision to
 #   create a graph that is restricted to those revisions.
 #
 #   This set of revisions includes not only revisions that directly
 #   touch files we're interested in, but also merges that merge two
 #   or more interesting revisions.
 
 class filemap_source(common.converter_source):
     def __init__(self, ui, baseconverter, filemap):
         super(filemap_source, self).__init__(ui)
         self.base = baseconverter
         self.filemapper = filemapper(ui, filemap)
         self.commits = {}
         # if a revision rev has parent p in the original revision graph, then
         # rev will have parent self.parentmap[p] in the restricted graph.
         self.parentmap = {}
         # self.wantedancestors[rev] is the set of all ancestors of rev that
         # are in the restricted graph.
         self.wantedancestors = {}
         self.convertedorder = None
         self._rebuilt = False
         self.origparents = {}
         self.children = {}
         self.seenchildren = {}
         # experimental config: convert.ignoreancestorcheck
         self.ignoreancestorcheck = self.ui.configbool('convert',
                                                       'ignoreancestorcheck')
 
     def before(self):
         self.base.before()
 
     def after(self):
         self.base.after()
 
     def setrevmap(self, revmap):
         # rebuild our state to make things restartable
         #
         # To avoid calling getcommit for every revision that has already
         # been converted, we rebuild only the parentmap, delaying the
         # rebuild of wantedancestors until we need it (i.e. until a
         # merge).
         #
         # We assume the order argument lists the revisions in
         # topological order, so that we can infer which revisions were
         # wanted by previous runs.
         self._rebuilt = not revmap
         seen = {SKIPREV: SKIPREV}
         dummyset = set()
         converted = []
         for rev in revmap.order:
             mapped = revmap[rev]
             wanted = mapped not in seen
             if wanted:
                 seen[mapped] = rev
                 self.parentmap[rev] = rev
             else:
                 self.parentmap[rev] = seen[mapped]
             self.wantedancestors[rev] = dummyset
             arg = seen[mapped]
             if arg == SKIPREV:
                 arg = None
             converted.append((rev, wanted, arg))
         self.convertedorder = converted
         return self.base.setrevmap(revmap)
 
     def rebuild(self):
         if self._rebuilt:
             return True
         self._rebuilt = True
         self.parentmap.clear()
         self.wantedancestors.clear()
         self.seenchildren.clear()
         for rev, wanted, arg in self.convertedorder:
             if rev not in self.origparents:
                 try:
                     self.origparents[rev] = self.getcommit(rev).parents
                 except error.RepoLookupError:
                     self.ui.debug("unknown revmap source: %s\n" % rev)
                     continue
             if arg is not None:
                 self.children[arg] = self.children.get(arg, 0) + 1
 
         for rev, wanted, arg in self.convertedorder:
             try:
                 parents = self.origparents[rev]
             except KeyError:
                 continue # unknown revmap source
             if wanted:
                 self.mark_wanted(rev, parents)
             else:
                 self.mark_not_wanted(rev, arg)
             self._discard(arg, *parents)
 
         return True
 
     def getheads(self):
         return self.base.getheads()
 
     def getcommit(self, rev):
         # We want to save a reference to the commit objects to be able
         # to rewrite their parents later on.
         c = self.commits[rev] = self.base.getcommit(rev)
         for p in c.parents:
             self.children[p] = self.children.get(p, 0) + 1
         return c
 
     def _cachedcommit(self, rev):
         if rev in self.commits:
             return self.commits[rev]
         return self.base.getcommit(rev)
 
     def _discard(self, *revs):
         for r in revs:
             if r is None:
                 continue
             self.seenchildren[r] = self.seenchildren.get(r, 0) + 1
             if self.seenchildren[r] == self.children[r]:
                 self.wantedancestors.pop(r, None)
                 self.parentmap.pop(r, None)
                 del self.seenchildren[r]
                 if self._rebuilt:
                     del self.children[r]
 
     def wanted(self, rev, i):
         # Return True if we're directly interested in rev.
         #
         # i is an index selecting one of the parents of rev (if rev
         # has no parents, i is None).  getchangedfiles will give us
         # the list of files that are different in rev and in the parent
         # indicated by i.  If we're interested in any of these files,
         # we're interested in rev.
         try:
             files = self.base.getchangedfiles(rev, i)
         except NotImplementedError:
             raise error.Abort(_("source repository doesn't support --filemap"))
         for f in files:
             if self.filemapper(f):
                 return True
         return False
 
     def mark_not_wanted(self, rev, p):
         # Mark rev as not interesting and update data structures.
 
         if p is None:
             # A root revision. Use SKIPREV to indicate that it doesn't
             # map to any revision in the restricted graph.  Put SKIPREV
             # in the set of wanted ancestors to simplify code elsewhere
             self.parentmap[rev] = SKIPREV
             self.wantedancestors[rev] = set((SKIPREV,))
             return
 
         # Reuse the data from our parent.
         self.parentmap[rev] = self.parentmap[p]
         self.wantedancestors[rev] = self.wantedancestors[p]
 
     def mark_wanted(self, rev, parents):
         # Mark rev ss wanted and update data structures.
 
         # rev will be in the restricted graph, so children of rev in
         # the original graph should still have rev as a parent in the
         # restricted graph.
         self.parentmap[rev] = rev
 
         # The set of wanted ancestors of rev is the union of the sets
         # of wanted ancestors of its parents. Plus rev itself.
         wrev = set()
         for p in parents:
             if p in self.wantedancestors:
                 wrev.update(self.wantedancestors[p])
             else:
                 self.ui.warn(_('warning: %s parent %s is missing\n') %
                              (rev, p))
         wrev.add(rev)
         self.wantedancestors[rev] = wrev
 
     def getchanges(self, rev, full):
         parents = self.commits[rev].parents
         if len(parents) > 1 and not self.ignoreancestorcheck:
             self.rebuild()
 
         # To decide whether we're interested in rev we:
         #
         # - calculate what parents rev will have if it turns out we're
         #   interested in it.  If it's going to have more than 1 parent,
         #   we're interested in it.
         #
         # - otherwise, we'll compare it with the single parent we found.
         #   If any of the files we're interested in is different in the
         #   the two revisions, we're interested in rev.
 
         # A parent p is interesting if its mapped version (self.parentmap[p]):
         # - is not SKIPREV
         # - is still not in the list of parents (we don't want duplicates)
         # - is not an ancestor of the mapped versions of the other parents or
         #   there is no parent in the same branch than the current revision.
         mparents = []
         knownparents = set()
         branch = self.commits[rev].branch
         hasbranchparent = False
         for i, p1 in enumerate(parents):
             mp1 = self.parentmap[p1]
             if mp1 == SKIPREV or mp1 in knownparents:
                 continue
 
             isancestor = (not self.ignoreancestorcheck and
                           any(p2 for p2 in parents
                               if p1 != p2 and mp1 != self.parentmap[p2]
                                  and mp1 in self.wantedancestors[p2]))
             if not isancestor and not hasbranchparent and len(parents) > 1:
                 # This could be expensive, avoid unnecessary calls.
                 if self._cachedcommit(p1).branch == branch:
                     hasbranchparent = True
             mparents.append((p1, mp1, i, isancestor))
             knownparents.add(mp1)
         # Discard parents ancestors of other parents if there is a
         # non-ancestor one on the same branch than current revision.
         if hasbranchparent:
             mparents = [p for p in mparents if not p[3]]
         wp = None
         if mparents:
             wp = max(p[2] for p in mparents)
             mparents = [p[1] for p in mparents]
         elif parents:
             wp = 0
 
         self.origparents[rev] = parents
 
         closed = False
         if 'close' in self.commits[rev].extra:
             # A branch closing revision is only useful if one of its
             # parents belong to the branch being closed
             pbranches = [self._cachedcommit(p).branch for p in mparents]
             if branch in pbranches:
                 closed = True
 
         if len(mparents) < 2 and not closed and not self.wanted(rev, wp):
             # We don't want this revision.
             # Update our state and tell the convert process to map this
             # revision to the same revision its parent as mapped to.
             p = None
             if parents:
                 p = parents[wp]
             self.mark_not_wanted(rev, p)
             self.convertedorder.append((rev, False, p))
             self._discard(*parents)
             return self.parentmap[rev]
 
         # We want this revision.
         # Rewrite the parents of the commit object
         self.commits[rev].parents = mparents
         self.mark_wanted(rev, parents)
         self.convertedorder.append((rev, True, None))
         self._discard(*parents)
 
         # Get the real changes and do the filtering/mapping. To be
         # able to get the files later on in getfile, we hide the
         # original filename in the rev part of the return value.
         changes, copies, cleanp2 = self.base.getchanges(rev, full)
         files = {}
         ncleanp2 = set(cleanp2)
         for f, r in changes:
             newf = self.filemapper(f)
             if newf and (newf != f or newf not in files):
                 files[newf] = (f, r)
                 if newf != f:
                     ncleanp2.discard(f)
         files = sorted(files.items())
 
         ncopies = {}
         for c in copies:
             newc = self.filemapper(c)
             if newc:
                 newsource = self.filemapper(copies[c])
                 if newsource:
                     ncopies[newc] = newsource
 
         return files, ncopies, ncleanp2
 
     def targetfilebelongstosource(self, targetfilename):
         return self.filemapper.istargetfile(targetfilename)
 
     def getfile(self, name, rev):
         realname, realrev = rev
         return self.base.getfile(realname, realrev)
 
     def gettags(self):
         return self.base.gettags()
 
     def hasnativeorder(self):
         return self.base.hasnativeorder()
 
     def lookuprev(self, rev):
         return self.base.lookuprev(rev)
 
     def getbookmarks(self):
         return self.base.getbookmarks()
 
     def converted(self, rev, sinkrev):
         self.base.converted(rev, sinkrev)

hgext/convert/git.py

@@ -1,398 +1,399
 # git.py - git support for the convert extension
 #
 #  Copyright 2005-2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import os
+
+from mercurial.i18n import _
 from mercurial import (
     config,
     error,
     node as nodemod,
 )
-from mercurial.i18n import _
 
 from . import (
     common,
 )
 
 class submodule(object):
     def __init__(self, path, node, url):
         self.path = path
         self.node = node
         self.url = url
 
     def hgsub(self):
         return "%s = [git]%s" % (self.path, self.url)
 
     def hgsubstate(self):
         return "%s %s" % (self.node, self.path)
 
 class convert_git(common.converter_source, common.commandline):
     # Windows does not support GIT_DIR= construct while other systems
     # cannot remove environment variable. Just assume none have
     # both issues.
 
     def _gitcmd(self, cmd, *args, **kwargs):
         return cmd('--git-dir=%s' % self.path, *args, **kwargs)
 
     def gitrun0(self, *args, **kwargs):
         return self._gitcmd(self.run0, *args, **kwargs)
 
     def gitrun(self, *args, **kwargs):
         return self._gitcmd(self.run, *args, **kwargs)
 
     def gitrunlines0(self, *args, **kwargs):
         return self._gitcmd(self.runlines0, *args, **kwargs)
 
     def gitrunlines(self, *args, **kwargs):
         return self._gitcmd(self.runlines, *args, **kwargs)
 
     def gitpipe(self, *args, **kwargs):
         return self._gitcmd(self._run3, *args, **kwargs)
 
     def __init__(self, ui, path, revs=None):
         super(convert_git, self).__init__(ui, path, revs=revs)
         common.commandline.__init__(self, ui, 'git')
 
         # Pass an absolute path to git to prevent from ever being interpreted
         # as a URL
         path = os.path.abspath(path)
 
         if os.path.isdir(path + "/.git"):
             path += "/.git"
         if not os.path.exists(path + "/objects"):
             raise common.NoRepo(_("%s does not look like a Git repository") %
                                 path)
 
         # The default value (50) is based on the default for 'git diff'.
         similarity = ui.configint('convert', 'git.similarity', default=50)
         if similarity < 0 or similarity > 100:
             raise error.Abort(_('similarity must be between 0 and 100'))
         if similarity > 0:
             self.simopt = ['-C%d%%' % similarity]
             findcopiesharder = ui.configbool('convert', 'git.findcopiesharder',
                                              False)
             if findcopiesharder:
                 self.simopt.append('--find-copies-harder')
         else:
             self.simopt = []
 
         common.checktool('git', 'git')
 
         self.path = path
         self.submodules = []
 
         self.catfilepipe = self.gitpipe('cat-file', '--batch')
 
     def after(self):
         for f in self.catfilepipe:
             f.close()
 
     def getheads(self):
         if not self.revs:
             output, status = self.gitrun('rev-parse', '--branches', '--remotes')
             heads = output.splitlines()
             if status:
                 raise error.Abort(_('cannot retrieve git heads'))
         else:
             heads = []
             for rev in self.revs:
                 rawhead, ret = self.gitrun('rev-parse', '--verify', rev)
                 heads.append(rawhead[:-1])
                 if ret:
                     raise error.Abort(_('cannot retrieve git head "%s"') % rev)
         return heads
 
     def catfile(self, rev, type):
         if rev == nodemod.nullhex:
             raise IOError
         self.catfilepipe[0].write(rev+'\n')
         self.catfilepipe[0].flush()
         info = self.catfilepipe[1].readline().split()
         if info[1] != type:
             raise error.Abort(_('cannot read %r object at %s') % (type, rev))
         size = int(info[2])
         data = self.catfilepipe[1].read(size)
         if len(data) < size:
             raise error.Abort(_('cannot read %r object at %s: unexpected size')
                              % (type, rev))
         # read the trailing newline
         self.catfilepipe[1].read(1)
         return data
 
     def getfile(self, name, rev):
         if rev == nodemod.nullhex:
             return None, None
         if name == '.hgsub':
             data = '\n'.join([m.hgsub() for m in self.submoditer()])
             mode = ''
         elif name == '.hgsubstate':
             data = '\n'.join([m.hgsubstate() for m in self.submoditer()])
             mode = ''
         else:
             data = self.catfile(rev, "blob")
             mode = self.modecache[(name, rev)]
         return data, mode
 
     def submoditer(self):
         null = nodemod.nullhex
         for m in sorted(self.submodules, key=lambda p: p.path):
             if m.node != null:
                 yield m
 
     def parsegitmodules(self, content):
         """Parse the formatted .gitmodules file, example file format:
         [submodule "sub"]\n
         \tpath = sub\n
         \turl = git://giturl\n
         """
         self.submodules = []
         c = config.config()
         # Each item in .gitmodules starts with whitespace that cant be parsed
         c.parse('.gitmodules', '\n'.join(line.strip() for line in
                                content.split('\n')))
         for sec in c.sections():
             s = c[sec]
             if 'url' in s and 'path' in s:
                 self.submodules.append(submodule(s['path'], '', s['url']))
 
     def retrievegitmodules(self, version):
         modules, ret = self.gitrun('show', '%s:%s' % (version, '.gitmodules'))
         if ret:
             # This can happen if a file is in the repo that has permissions
             # 160000, but there is no .gitmodules file.
             self.ui.warn(_("warning: cannot read submodules config file in "
                            "%s\n") % version)
             return
 
         try:
             self.parsegitmodules(modules)
         except error.ParseError:
             self.ui.warn(_("warning: unable to parse .gitmodules in %s\n")
                          % version)
             return
 
         for m in self.submodules:
             node, ret = self.gitrun('rev-parse', '%s:%s' % (version, m.path))
             if ret:
                 continue
             m.node = node.strip()
 
     def getchanges(self, version, full):
         if full:
             raise error.Abort(_("convert from git does not support --full"))
         self.modecache = {}
         cmd = ['diff-tree','-z', '--root', '-m', '-r'] + self.simopt + [version]
         output, status = self.gitrun(*cmd)
         if status:
             raise error.Abort(_('cannot read changes in %s') % version)
         changes = []
         copies = {}
         seen = set()
         entry = None
         subexists = [False]
         subdeleted = [False]
         difftree = output.split('\x00')
         lcount = len(difftree)
         i = 0
 
         skipsubmodules = self.ui.configbool('convert', 'git.skipsubmodules',
                                             False)
         def add(entry, f, isdest):
             seen.add(f)
             h = entry[3]
             p = (entry[1] == "100755")
             s = (entry[1] == "120000")
             renamesource = (not isdest and entry[4][0] == 'R')
 
             if f == '.gitmodules':
                 if skipsubmodules:
                     return
 
                 subexists[0] = True
                 if entry[4] == 'D' or renamesource:
                     subdeleted[0] = True
                     changes.append(('.hgsub', nodemod.nullhex))
                 else:
                     changes.append(('.hgsub', ''))
             elif entry[1] == '160000' or entry[0] == ':160000':
                 if not skipsubmodules:
                     subexists[0] = True
             else:
                 if renamesource:
                     h = nodemod.nullhex
                 self.modecache[(f, h)] = (p and "x") or (s and "l") or ""
                 changes.append((f, h))
 
         while i < lcount:
             l = difftree[i]
             i += 1
             if not entry:
                 if not l.startswith(':'):
                     continue
                 entry = l.split()
                 continue
             f = l
             if entry[4][0] == 'C':
                 copysrc = f
                 copydest = difftree[i]
                 i += 1
                 f = copydest
                 copies[copydest] = copysrc
             if f not in seen:
                 add(entry, f, False)
             # A file can be copied multiple times, or modified and copied
             # simultaneously. So f can be repeated even if fdest isn't.
             if entry[4][0] == 'R':
                 # rename: next line is the destination
                 fdest = difftree[i]
                 i += 1
                 if fdest not in seen:
                     add(entry, fdest, True)
                     # .gitmodules isn't imported at all, so it being copied to
                     # and fro doesn't really make sense
                     if f != '.gitmodules' and fdest != '.gitmodules':
                         copies[fdest] = f
             entry = None
 
         if subexists[0]:
             if subdeleted[0]:
                 changes.append(('.hgsubstate', nodemod.nullhex))
             else:
                 self.retrievegitmodules(version)
                 changes.append(('.hgsubstate', ''))
         return (changes, copies, set())
 
     def getcommit(self, version):
         c = self.catfile(version, "commit") # read the commit hash
         end = c.find("\n\n")
         message = c[end + 2:]
         message = self.recode(message)
         l = c[:end].splitlines()
         parents = []
         author = committer = None
         for e in l[1:]:
             n, v = e.split(" ", 1)
             if n == "author":
                 p = v.split()
                 tm, tz = p[-2:]
                 author = " ".join(p[:-2])
                 if author[0] == "<": author = author[1:-1]
                 author = self.recode(author)
             if n == "committer":
                 p = v.split()
                 tm, tz = p[-2:]
                 committer = " ".join(p[:-2])
                 if committer[0] == "<": committer = committer[1:-1]
                 committer = self.recode(committer)
             if n == "parent":
                 parents.append(v)
 
         if committer and committer != author:
             message += "\ncommitter: %s\n" % committer
         tzs, tzh, tzm = tz[-5:-4] + "1", tz[-4:-2], tz[-2:]
         tz = -int(tzs) * (int(tzh) * 3600 + int(tzm))
         date = tm + " " + str(tz)
 
         c = common.commit(parents=parents, date=date, author=author,
                           desc=message,
                           rev=version)
         return c
 
     def numcommits(self):
         output, ret = self.gitrunlines('rev-list', '--all')
         if ret:
             raise error.Abort(_('cannot retrieve number of commits in %s') \
                               % self.path)
         return len(output)
 
     def gettags(self):
         tags = {}
         alltags = {}
         output, status = self.gitrunlines('ls-remote', '--tags', self.path)
 
         if status:
             raise error.Abort(_('cannot read tags from %s') % self.path)
         prefix = 'refs/tags/'
 
         # Build complete list of tags, both annotated and bare ones
         for line in output:
             line = line.strip()
             if line.startswith("error:") or line.startswith("fatal:"):
                 raise error.Abort(_('cannot read tags from %s') % self.path)
             node, tag = line.split(None, 1)
             if not tag.startswith(prefix):
                 continue
             alltags[tag[len(prefix):]] = node
 
         # Filter out tag objects for annotated tag refs
         for tag in alltags:
             if tag.endswith('^{}'):
                 tags[tag[:-3]] = alltags[tag]
             else:
                 if tag + '^{}' in alltags:
                     continue
                 else:
                     tags[tag] = alltags[tag]
 
         return tags
 
     def getchangedfiles(self, version, i):
         changes = []
         if i is None:
             output, status = self.gitrunlines('diff-tree', '--root', '-m',
                                               '-r', version)
             if status:
                 raise error.Abort(_('cannot read changes in %s') % version)
             for l in output:
                 if "\t" not in l:
                     continue
                 m, f = l[:-1].split("\t")
                 changes.append(f)
         else:
             output, status = self.gitrunlines('diff-tree', '--name-only',
                                               '--root', '-r', version,
                                               '%s^%s' % (version, i + 1), '--')
             if status:
                 raise error.Abort(_('cannot read changes in %s') % version)
             changes = [f.rstrip('\n') for f in output]
 
         return changes
 
     def getbookmarks(self):
         bookmarks = {}
 
         # Handle local and remote branches
         remoteprefix = self.ui.config('convert', 'git.remoteprefix', 'remote')
         reftypes = [
             # (git prefix, hg prefix)
             ('refs/remotes/origin/', remoteprefix + '/'),
             ('refs/heads/', '')
         ]
 
         exclude = set([
             'refs/remotes/origin/HEAD',
         ])
 
         try:
             output, status = self.gitrunlines('show-ref')
             for line in output:
                 line = line.strip()
                 rev, name = line.split(None, 1)
                 # Process each type of branch
                 for gitprefix, hgprefix in reftypes:
                     if not name.startswith(gitprefix) or name in exclude:
                         continue
                     name = '%s%s' % (hgprefix, name[len(gitprefix):])
                     bookmarks[name] = rev
         except Exception:
             pass
 
         return bookmarks
 
     def checkrevformat(self, revstr, mapname='splicemap'):
         """ git revision string is a 40 byte hex """
         self.checkhexformat(revstr, mapname)

hgext/convert/gnuarch.py

@@ -1,351 +1,352
 # gnuarch.py - GNU Arch support for the convert extension
 #
 #  Copyright 2008, 2009 Aleix Conchillo Flaque <[e-mail]>
 #  and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import email
 import os
 import shutil
 import stat
 import tempfile
+
+from mercurial.i18n import _
 from mercurial import (
     encoding,
     error,
     util,
 )
-from mercurial.i18n import _
 from . import common
 
 class gnuarch_source(common.converter_source, common.commandline):
 
     class gnuarch_rev(object):
         def __init__(self, rev):
             self.rev = rev
             self.summary = ''
             self.date = None
             self.author = ''
             self.continuationof = None
             self.add_files = []
             self.mod_files = []
             self.del_files = []
             self.ren_files = {}
             self.ren_dirs = {}
 
     def __init__(self, ui, path, revs=None):
         super(gnuarch_source, self).__init__(ui, path, revs=revs)
 
         if not os.path.exists(os.path.join(path, '{arch}')):
             raise common.NoRepo(_("%s does not look like a GNU Arch repository")
                          % path)
 
         # Could use checktool, but we want to check for baz or tla.
         self.execmd = None
         if util.findexe('baz'):
             self.execmd = 'baz'
         else:
             if util.findexe('tla'):
                 self.execmd = 'tla'
             else:
                 raise error.Abort(_('cannot find a GNU Arch tool'))
 
         common.commandline.__init__(self, ui, self.execmd)
 
         self.path = os.path.realpath(path)
         self.tmppath = None
 
         self.treeversion = None
         self.lastrev = None
         self.changes = {}
         self.parents = {}
         self.tags = {}
         self.catlogparser = email.Parser.Parser()
         self.encoding = encoding.encoding
         self.archives = []
 
     def before(self):
         # Get registered archives
         self.archives = [i.rstrip('\n')
                          for i in self.runlines0('archives', '-n')]
 
         if self.execmd == 'tla':
             output = self.run0('tree-version', self.path)
         else:
             output = self.run0('tree-version', '-d', self.path)
         self.treeversion = output.strip()
 
         # Get name of temporary directory
         version = self.treeversion.split('/')
         self.tmppath = os.path.join(tempfile.gettempdir(),
                                     'hg-%s' % version[1])
 
         # Generate parents dictionary
         self.parents[None] = []
         treeversion = self.treeversion
         child = None
         while treeversion:
             self.ui.status(_('analyzing tree version %s...\n') % treeversion)
 
             archive = treeversion.split('/')[0]
             if archive not in self.archives:
                 self.ui.status(_('tree analysis stopped because it points to '
                                  'an unregistered archive %s...\n') % archive)
                 break
 
             # Get the complete list of revisions for that tree version
             output, status = self.runlines('revisions', '-r', '-f', treeversion)
             self.checkexit(status, 'failed retrieving revisions for %s'
                            % treeversion)
 
             # No new iteration unless a revision has a continuation-of header
             treeversion = None
 
             for l in output:
                 rev = l.strip()
                 self.changes[rev] = self.gnuarch_rev(rev)
                 self.parents[rev] = []
 
                 # Read author, date and summary
                 catlog, status = self.run('cat-log', '-d', self.path, rev)
                 if status:
                     catlog  = self.run0('cat-archive-log', rev)
                 self._parsecatlog(catlog, rev)
 
                 # Populate the parents map
                 self.parents[child].append(rev)
 
                 # Keep track of the current revision as the child of the next
                 # revision scanned
                 child = rev
 
                 # Check if we have to follow the usual incremental history
                 # or if we have to 'jump' to a different treeversion given
                 # by the continuation-of header.
                 if self.changes[rev].continuationof:
                     treeversion = '--'.join(
                         self.changes[rev].continuationof.split('--')[:-1])
                     break
 
                 # If we reached a base-0 revision w/o any continuation-of
                 # header, it means the tree history ends here.
                 if rev[-6:] == 'base-0':
                     break
 
     def after(self):
         self.ui.debug('cleaning up %s\n' % self.tmppath)
         shutil.rmtree(self.tmppath, ignore_errors=True)
 
     def getheads(self):
         return self.parents[None]
 
     def getfile(self, name, rev):
         if rev != self.lastrev:
             raise error.Abort(_('internal calling inconsistency'))
 
         if not os.path.lexists(os.path.join(self.tmppath, name)):
             return None, None
 
         return self._getfile(name, rev)
 
     def getchanges(self, rev, full):
         if full:
             raise error.Abort(_("convert from arch does not support --full"))
         self._update(rev)
         changes = []
         copies = {}
 
         for f in self.changes[rev].add_files:
             changes.append((f, rev))
 
         for f in self.changes[rev].mod_files:
             changes.append((f, rev))
 
         for f in self.changes[rev].del_files:
             changes.append((f, rev))
 
         for src in self.changes[rev].ren_files:
             to = self.changes[rev].ren_files[src]
             changes.append((src, rev))
             changes.append((to, rev))
             copies[to] = src
 
         for src in self.changes[rev].ren_dirs:
             to = self.changes[rev].ren_dirs[src]
             chgs, cps = self._rendirchanges(src, to)
             changes += [(f, rev) for f in chgs]
             copies.update(cps)
 
         self.lastrev = rev
         return sorted(set(changes)), copies, set()
 
     def getcommit(self, rev):
         changes = self.changes[rev]
         return common.commit(author=changes.author, date=changes.date,
                              desc=changes.summary, parents=self.parents[rev],
                              rev=rev)
 
     def gettags(self):
         return self.tags
 
     def _execute(self, cmd, *args, **kwargs):
         cmdline = [self.execmd, cmd]
         cmdline += args
         cmdline = [util.shellquote(arg) for arg in cmdline]
         cmdline += ['>', os.devnull, '2>', os.devnull]
         cmdline = util.quotecommand(' '.join(cmdline))
         self.ui.debug(cmdline, '\n')
         return os.system(cmdline)
 
     def _update(self, rev):
         self.ui.debug('applying revision %s...\n' % rev)
         changeset, status = self.runlines('replay', '-d', self.tmppath,
                                               rev)
         if status:
             # Something went wrong while merging (baz or tla
             # issue?), get latest revision and try from there
             shutil.rmtree(self.tmppath, ignore_errors=True)
             self._obtainrevision(rev)
         else:
             old_rev = self.parents[rev][0]
             self.ui.debug('computing changeset between %s and %s...\n'
                           % (old_rev, rev))
             self._parsechangeset(changeset, rev)
 
     def _getfile(self, name, rev):
         mode = os.lstat(os.path.join(self.tmppath, name)).st_mode
         if stat.S_ISLNK(mode):
             data = os.readlink(os.path.join(self.tmppath, name))
             if mode:
                 mode = 'l'
             else:
                 mode = ''
         else:
             data = open(os.path.join(self.tmppath, name), 'rb').read()
             mode = (mode & 0o111) and 'x' or ''
         return data, mode
 
     def _exclude(self, name):
         exclude = ['{arch}', '.arch-ids', '.arch-inventory']
         for exc in exclude:
             if name.find(exc) != -1:
                 return True
         return False
 
     def _readcontents(self, path):
         files = []
         contents = os.listdir(path)
         while len(contents) > 0:
             c = contents.pop()
             p = os.path.join(path, c)
             # os.walk could be used, but here we avoid internal GNU
             # Arch files and directories, thus saving a lot time.
             if not self._exclude(p):
                 if os.path.isdir(p):
                     contents += [os.path.join(c, f) for f in os.listdir(p)]
                 else:
                     files.append(c)
         return files
 
     def _rendirchanges(self, src, dest):
         changes = []
         copies = {}
         files = self._readcontents(os.path.join(self.tmppath, dest))
         for f in files:
             s = os.path.join(src, f)
             d = os.path.join(dest, f)
             changes.append(s)
             changes.append(d)
             copies[d] = s
         return changes, copies
 
     def _obtainrevision(self, rev):
         self.ui.debug('obtaining revision %s...\n' % rev)
         output = self._execute('get', rev, self.tmppath)
         self.checkexit(output)
         self.ui.debug('analyzing revision %s...\n' % rev)
         files = self._readcontents(self.tmppath)
         self.changes[rev].add_files += files
 
     def _stripbasepath(self, path):
         if path.startswith('./'):
             return path[2:]
         return path
 
     def _parsecatlog(self, data, rev):
         try:
             catlog = self.catlogparser.parsestr(data)
 
             # Commit date
             self.changes[rev].date = util.datestr(
                 util.strdate(catlog['Standard-date'],
                              '%Y-%m-%d %H:%M:%S'))
 
             # Commit author
             self.changes[rev].author = self.recode(catlog['Creator'])
 
             # Commit description
             self.changes[rev].summary = '\n\n'.join((catlog['Summary'],
                                                     catlog.get_payload()))
             self.changes[rev].summary = self.recode(self.changes[rev].summary)
 
             # Commit revision origin when dealing with a branch or tag
             if 'Continuation-of' in catlog:
                 self.changes[rev].continuationof = self.recode(
                     catlog['Continuation-of'])
         except Exception:
             raise error.Abort(_('could not parse cat-log of %s') % rev)
 
     def _parsechangeset(self, data, rev):
         for l in data:
             l = l.strip()
             # Added file (ignore added directory)
             if l.startswith('A') and not l.startswith('A/'):
                 file = self._stripbasepath(l[1:].strip())
                 if not self._exclude(file):
                     self.changes[rev].add_files.append(file)
             # Deleted file (ignore deleted directory)
             elif l.startswith('D') and not l.startswith('D/'):
                 file = self._stripbasepath(l[1:].strip())
                 if not self._exclude(file):
                     self.changes[rev].del_files.append(file)
             # Modified binary file
             elif l.startswith('Mb'):
                 file = self._stripbasepath(l[2:].strip())
                 if not self._exclude(file):
                     self.changes[rev].mod_files.append(file)
             # Modified link
             elif l.startswith('M->'):
                 file = self._stripbasepath(l[3:].strip())
                 if not self._exclude(file):
                     self.changes[rev].mod_files.append(file)
             # Modified file
             elif l.startswith('M'):
                 file = self._stripbasepath(l[1:].strip())
                 if not self._exclude(file):
                     self.changes[rev].mod_files.append(file)
             # Renamed file (or link)
             elif l.startswith('=>'):
                 files = l[2:].strip().split(' ')
                 if len(files) == 1:
                     files = l[2:].strip().split('\t')
                 src = self._stripbasepath(files[0])
                 dst = self._stripbasepath(files[1])
                 if not self._exclude(src) and not self._exclude(dst):
                     self.changes[rev].ren_files[src] = dst
             # Conversion from file to link or from link to file (modified)
             elif l.startswith('ch'):
                 file = self._stripbasepath(l[2:].strip())
                 if not self._exclude(file):
                     self.changes[rev].mod_files.append(file)
             # Renamed directory
             elif l.startswith('/>'):
                 dirs = l[2:].strip().split(' ')
                 if len(dirs) == 1:
                     dirs = l[2:].strip().split('\t')
                 src = self._stripbasepath(dirs[0])
                 dst = self._stripbasepath(dirs[1])
                 if not self._exclude(src) and not self._exclude(dst):
                     self.changes[rev].ren_dirs[src] = dst