subprocess: use subprocessio helper to run various subprocess commands.

vcsserver/hooks.py

@@ -1,484 +1,467 b''
 # -*- coding: utf-8 -*-
 
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2018 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 import os
 import sys
 import json
 import logging
 import collections
 import importlib
 
 from httplib import HTTPConnection
 
 
 import mercurial.scmutil
 import mercurial.node
 import simplejson as json
 
 from vcsserver import exceptions, subprocessio, settings
 
 log = logging.getLogger(__name__)
 
 
 class HooksHttpClient(object):
     connection = None
 
     def __init__(self, hooks_uri):
         self.hooks_uri = hooks_uri
 
     def __call__(self, method, extras):
         connection = HTTPConnection(self.hooks_uri)
         body = self._serialize(method, extras)
         connection.request('POST', '/', body)
         response = connection.getresponse()
         return json.loads(response.read())
 
     def _serialize(self, hook_name, extras):
         data = {
             'method': hook_name,
             'extras': extras
         }
         return json.dumps(data)
 
 
 class HooksDummyClient(object):
     def __init__(self, hooks_module):
         self._hooks_module = importlib.import_module(hooks_module)
 
     def __call__(self, hook_name, extras):
         with self._hooks_module.Hooks() as hooks:
             return getattr(hooks, hook_name)(extras)
 
 
 class RemoteMessageWriter(object):
     """Writer base class."""
     def write(self, message):
         raise NotImplementedError()
 
 
 class HgMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to mercurial clients."""
 
     def __init__(self, ui):
         self.ui = ui
 
     def write(self, message):
         # TODO: Check why the quiet flag is set by default.
         old = self.ui.quiet
         self.ui.quiet = False
         self.ui.status(message.encode('utf-8'))
         self.ui.quiet = old
 
 
 class GitMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to git clients."""
 
     def __init__(self, stdout=None):
         self.stdout = stdout or sys.stdout
 
     def write(self, message):
         self.stdout.write(message.encode('utf-8'))
 
 
 def _handle_exception(result):
     exception_class = result.get('exception')
     exception_traceback = result.get('exception_traceback')
 
     if exception_traceback:
         log.error('Got traceback from remote call:%s', exception_traceback)
 
     if exception_class == 'HTTPLockedRC':
         raise exceptions.RepositoryLockedException(*result['exception_args'])
     elif exception_class == 'RepositoryError':
         raise exceptions.VcsException(*result['exception_args'])
     elif exception_class:
         raise Exception('Got remote exception "%s" with args "%s"' %
                         (exception_class, result['exception_args']))
 
 
 def _get_hooks_client(extras):
     if 'hooks_uri' in extras:
         protocol = extras.get('hooks_protocol')
         return HooksHttpClient(extras['hooks_uri'])
     else:
         return HooksDummyClient(extras['hooks_module'])
 
 
 def _call_hook(hook_name, extras, writer):
     hooks = _get_hooks_client(extras)
     result = hooks(hook_name, extras)
     log.debug('Hooks got result: %s', result)
     writer.write(result['output'])
     _handle_exception(result)
 
     return result['status']
 
 
 def _extras_from_ui(ui):
     hook_data = ui.config('rhodecode', 'RC_SCM_DATA')
     if not hook_data:
         # maybe it's inside environ ?
         env_hook_data = os.environ.get('RC_SCM_DATA')
         if env_hook_data:
             hook_data = env_hook_data
 
     extras = {}
     if hook_data:
         extras = json.loads(hook_data)
     return extras
 
 
 def _rev_range_hash(repo, node):
 
     commits = []
     for rev in xrange(repo[node], len(repo)):
         ctx = repo[rev]
         commit_id = mercurial.node.hex(ctx.node())
         branch = ctx.branch()
         commits.append((commit_id, branch))
 
     return commits
 
 
 def repo_size(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('repo_size', extras, HgMessageWriter(ui))
 
 
 def pre_pull(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('pre_pull', extras, HgMessageWriter(ui))
 
 
 def pre_pull_ssh(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     if extras and extras.get('SSH'):
         return pre_pull(ui, repo, **kwargs)
     return 0
 
 
 def post_pull(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     return _call_hook('post_pull', extras, HgMessageWriter(ui))
 
 
 def post_pull_ssh(ui, repo, **kwargs):
     extras = _extras_from_ui(ui)
     if extras and extras.get('SSH'):
         return post_pull(ui, repo, **kwargs)
     return 0
 
 
 def pre_push(ui, repo, node=None, **kwargs):
     extras = _extras_from_ui(ui)
 
     rev_data = []
     if node and kwargs.get('hooktype') == 'pretxnchangegroup':
         branches = collections.defaultdict(list)
         for commit_id, branch in _rev_range_hash(repo, node):
             branches[branch].append(commit_id)
 
         for branch, commits in branches.iteritems():
             old_rev = kwargs.get('node_last') or commits[0]
             rev_data.append({
                 'old_rev': old_rev,
                 'new_rev': commits[-1],
                 'ref': '',
                 'type': 'branch',
                 'name': branch,
             })
 
     extras['commit_ids'] = rev_data
     return _call_hook('pre_push', extras, HgMessageWriter(ui))
 
 
 def pre_push_ssh(ui, repo, node=None, **kwargs):
     if _extras_from_ui(ui).get('SSH'):
         return pre_push(ui, repo, node, **kwargs)
 
     return 0
 
 
 def pre_push_ssh_auth(ui, repo, node=None, **kwargs):
     extras = _extras_from_ui(ui)
     if extras.get('SSH'):
         permission = extras['SSH_PERMISSIONS']
 
         if 'repository.write' == permission or 'repository.admin' == permission:
             return 0
 
         # non-zero ret code
         return 1
 
     return 0
 
 
 def post_push(ui, repo, node, **kwargs):
     extras = _extras_from_ui(ui)
 
     commit_ids = []
     branches = []
     bookmarks = []
     tags = []
 
     for commit_id, branch in _rev_range_hash(repo, node):
         commit_ids.append(commit_id)
         if branch not in branches:
             branches.append(branch)
 
     if hasattr(ui, '_rc_pushkey_branches'):
         bookmarks = ui._rc_pushkey_branches
 
     extras['commit_ids'] = commit_ids
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': bookmarks,
         'tags': tags
     }
 
     return _call_hook('post_push', extras, HgMessageWriter(ui))
 
 
 def post_push_ssh(ui, repo, node, **kwargs):
     if _extras_from_ui(ui).get('SSH'):
         return post_push(ui, repo, node, **kwargs)
     return 0
 
 
 def key_push(ui, repo, **kwargs):
     if kwargs['new'] != '0' and kwargs['namespace'] == 'bookmarks':
         # store new bookmarks in our UI object propagated later to post_push
         ui._rc_pushkey_branches = repo[kwargs['key']].bookmarks()
     return
 
 
 # backward compat
 log_pull_action = post_pull
 
 # backward compat
 log_push_action = post_push
 
 
 def handle_git_pre_receive(unused_repo_path, unused_revs, unused_env):
     """
     Old hook name: keep here for backward compatibility.
 
     This is only required when the installed git hooks are not upgraded.
     """
     pass
 
 
 def handle_git_post_receive(unused_repo_path, unused_revs, unused_env):
     """
     Old hook name: keep here for backward compatibility.
 
     This is only required when the installed git hooks are not upgraded.
     """
     pass
 
 
 HookResponse = collections.namedtuple('HookResponse', ('status', 'output'))
 
 
 def git_pre_pull(extras):
     """
     Pre pull hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     if 'pull' not in extras['hooks']:
         return HookResponse(0, '')
 
     stdout = io.BytesIO()
     try:
         status = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
     except Exception as error:
         status = 128
         stdout.write('ERROR: %s\n' % str(error))
 
     return HookResponse(status, stdout.getvalue())
 
 
 def git_post_pull(extras):
     """
     Post pull hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     if 'pull' not in extras['hooks']:
         return HookResponse(0, '')
 
     stdout = io.BytesIO()
     try:
         status = _call_hook('post_pull', extras, GitMessageWriter(stdout))
     except Exception as error:
         status = 128
         stdout.write('ERROR: %s\n' % error)
 
     return HookResponse(status, stdout.getvalue())
 
 
 def _parse_git_ref_lines(revision_lines):
     rev_data = []
     for revision_line in revision_lines or []:
         old_rev, new_rev, ref = revision_line.strip().split(' ')
         ref_data = ref.split('/', 2)
         if ref_data[1] in ('tags', 'heads'):
             rev_data.append({
                 'old_rev': old_rev,
                 'new_rev': new_rev,
                 'ref': ref,
                 'type': ref_data[1],
                 'name': ref_data[2],
             })
     return rev_data
 
 
 def git_pre_receive(unused_repo_path, revision_lines, env):
     """
     Pre push hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     extras = json.loads(env['RC_SCM_DATA'])
     rev_data = _parse_git_ref_lines(revision_lines)
     if 'push' not in extras['hooks']:
         return 0
     extras['commit_ids'] = rev_data
     return _call_hook('pre_push', extras, GitMessageWriter())
 
 
-def _run_command(arguments):
-    """
-    Run the specified command and return the stdout.
-
-    :param arguments: sequence of program arguments (including the program name)
-    :type arguments: list[str]
-    """
-
-    cmd = arguments
-    try:
-        gitenv = os.environ.copy()
-        _opts = {'env': gitenv, 'shell': False, 'fail_on_stderr': False}
-        p = subprocessio.SubprocessIOChunker(cmd, **_opts)
-        stdout = ''.join(p)
-    except (EnvironmentError, OSError) as err:
-        cmd = ' '.join(cmd)  # human friendly CMD
-        tb_err = ("Couldn't run git command (%s).\n"
-                  "Original error was:%s\n" % (cmd, err))
-        log.exception(tb_err)
-        raise Exception(tb_err)
-
-    return stdout
-
-
 def git_post_receive(unused_repo_path, revision_lines, env):
     """
     Post push hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     extras = json.loads(env['RC_SCM_DATA'])
     if 'push' not in extras['hooks']:
         return 0
 
     rev_data = _parse_git_ref_lines(revision_lines)
 
     git_revs = []
 
     # N.B.(skreft): it is ok to just call git, as git before calling a
     # subcommand sets the PATH environment variable so that it point to the
     # correct version of the git executable.
     empty_commit_id = '0' * 40
     branches = []
     tags = []
     for push_ref in rev_data:
         type_ = push_ref['type']
 
         if type_ == 'heads':
             if push_ref['old_rev'] == empty_commit_id:
                 # starting new branch case
                 if push_ref['name'] not in branches:
                     branches.append(push_ref['name'])
 
                 # Fix up head revision if needed
                 cmd = [settings.GIT_EXECUTABLE, 'show', 'HEAD']
                 try:
-                    _run_command(cmd)
+                    subprocessio.run_command(cmd, env=os.environ.copy())
                 except Exception:
                     cmd = [settings.GIT_EXECUTABLE, 'symbolic-ref', 'HEAD',
                            'refs/heads/%s' % push_ref['name']]
                     print("Setting default branch to %s" % push_ref['name'])
-                    _run_command(cmd)
+                    subprocessio.run_command(cmd, env=os.environ.copy())
 
-                cmd = [settings.GIT_EXECUTABLE, 'for-each-ref', '--format=%(refname)',
-                       'refs/heads/*']
-                heads = _run_command(cmd)
+                cmd = [settings.GIT_EXECUTABLE, 'for-each-ref',
+                       '--format=%(refname)', 'refs/heads/*']
+                stdout, stderr = subprocessio.run_command(
+                    cmd, env=os.environ.copy())
+                heads = stdout
                 heads = heads.replace(push_ref['ref'], '')
                 heads = ' '.join(head for head in heads.splitlines() if head)
-                cmd = [settings.GIT_EXECUTABLE, 'log', '--reverse', '--pretty=format:%H',
-                        '--', push_ref['new_rev'], '--not', heads]
-                git_revs.extend(_run_command(cmd).splitlines())
+                cmd = [settings.GIT_EXECUTABLE, 'log', '--reverse',
+                       '--pretty=format:%H', '--', push_ref['new_rev'],
+                       '--not', heads]
+                stdout, stderr = subprocessio.run_command(
+                    cmd, env=os.environ.copy())
+                git_revs.extend(stdout.splitlines())
             elif push_ref['new_rev'] == empty_commit_id:
                 # delete branch case
                 git_revs.append('delete_branch=>%s' % push_ref['name'])
             else:
                 if push_ref['name'] not in branches:
                     branches.append(push_ref['name'])
 
                 cmd = [settings.GIT_EXECUTABLE, 'log',
                        '{old_rev}..{new_rev}'.format(**push_ref),
                        '--reverse', '--pretty=format:%H']
-                git_revs.extend(_run_command(cmd).splitlines())
+                stdout, stderr = subprocessio.run_command(
+                    cmd, env=os.environ.copy())
+                git_revs.extend(stdout.splitlines())
         elif type_ == 'tags':
             if push_ref['name'] not in tags:
                 tags.append(push_ref['name'])
             git_revs.append('tag=>%s' % push_ref['name'])
 
     extras['commit_ids'] = git_revs
     extras['new_refs'] = {
         'branches': branches,
         'bookmarks': [],
         'tags': tags,
     }
 
     if 'repo_size' in extras['hooks']:
         try:
             _call_hook('repo_size', extras, GitMessageWriter())
         except:
             pass
 
     return _call_hook('post_push', extras, GitMessageWriter())

vcsserver/subprocessio.py

@@ -1,484 +1,511 b''
 """
 Module provides a class allowing to wrap communication over subprocess.Popen
 input, output, error streams into a meaningfull, non-blocking, concurrent
 stream processor exposing the output data as an iterator fitting to be a
 return value passed by a WSGI applicaiton to a WSGI server per PEP 3333.
 
 Copyright (c) 2011  Daniel Dotsenko <dotsa[at]hotmail.com>
 
 This file is part of git_http_backend.py Project.
 
 git_http_backend.py Project is free software: you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public License as
 published by the Free Software Foundation, either version 2.1 of the License,
 or (at your option) any later version.
 
 git_http_backend.py Project is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 GNU Lesser General Public License for more details.
 
 You should have received a copy of the GNU Lesser General Public License
 along with git_http_backend.py Project.
 If not, see <http://www.gnu.org/licenses/>.
 """
 import os
 import logging
 import subprocess32 as subprocess
 from collections import deque
 from threading import Event, Thread
 
 log = logging.getLogger(__name__)
 
 
 class StreamFeeder(Thread):
     """
     Normal writing into pipe-like is blocking once the buffer is filled.
     This thread allows a thread to seep data from a file-like into a pipe
     without blocking the main thread.
     We close inpipe once the end of the source stream is reached.
     """
 
     def __init__(self, source):
         super(StreamFeeder, self).__init__()
         self.daemon = True
         filelike = False
         self.bytes = bytes()
         if type(source) in (type(''), bytes, bytearray):  # string-like
             self.bytes = bytes(source)
         else:  # can be either file pointer or file-like
             if type(source) in (int, long):  # file pointer it is
                 # converting file descriptor (int) stdin into file-like
                 try:
                     source = os.fdopen(source, 'rb', 16384)
                 except Exception:
                     pass
             # let's see if source is file-like by now
             try:
                 filelike = source.read
             except Exception:
                 pass
         if not filelike and not self.bytes:
             raise TypeError("StreamFeeder's source object must be a readable "
                             "file-like, a file descriptor, or a string-like.")
         self.source = source
         self.readiface, self.writeiface = os.pipe()
 
     def run(self):
         t = self.writeiface
         if self.bytes:
             os.write(t, self.bytes)
         else:
             s = self.source
             b = s.read(4096)
             while b:
                 os.write(t, b)
                 b = s.read(4096)
         os.close(t)
 
     @property
     def output(self):
         return self.readiface
 
 
 class InputStreamChunker(Thread):
     def __init__(self, source, target, buffer_size, chunk_size):
 
         super(InputStreamChunker, self).__init__()
 
         self.daemon = True  # die die die.
 
         self.source = source
         self.target = target
         self.chunk_count_max = int(buffer_size / chunk_size) + 1
         self.chunk_size = chunk_size
 
         self.data_added = Event()
         self.data_added.clear()
 
         self.keep_reading = Event()
         self.keep_reading.set()
 
         self.EOF = Event()
         self.EOF.clear()
 
         self.go = Event()
         self.go.set()
 
     def stop(self):
         self.go.clear()
         self.EOF.set()
         try:
             # this is not proper, but is done to force the reader thread let
             # go of the input because, if successful, .close() will send EOF
             # down the pipe.
             self.source.close()
         except:
             pass
 
     def run(self):
         s = self.source
         t = self.target
         cs = self.chunk_size
         chunk_count_max = self.chunk_count_max
         keep_reading = self.keep_reading
         da = self.data_added
         go = self.go
 
         try:
             b = s.read(cs)
         except ValueError:
             b = ''
 
         timeout_input = 20
         while b and go.is_set():
             if len(t) > chunk_count_max:
                 keep_reading.clear()
                 keep_reading.wait(timeout_input)
                 if len(t) > chunk_count_max + timeout_input:
                     log.error("Timed out while waiting for input from subprocess.")
                     os._exit(-1)  # this will cause the worker to recycle itself
 
             t.append(b)
             da.set()
             b = s.read(cs)
         self.EOF.set()
         da.set()  # for cases when done but there was no input.
 
 
 class BufferedGenerator(object):
     """
     Class behaves as a non-blocking, buffered pipe reader.
     Reads chunks of data (through a thread)
     from a blocking pipe, and attaches these to an array (Deque) of chunks.
     Reading is halted in the thread when max chunks is internally buffered.
     The .next() may operate in blocking or non-blocking fashion by yielding
     '' if no data is ready
     to be sent or by not returning until there is some data to send
     When we get EOF from underlying source pipe we raise the marker to raise
     StopIteration after the last chunk of data is yielded.
     """
 
     def __init__(self, source, buffer_size=65536, chunk_size=4096,
                  starting_values=None, bottomless=False):
         starting_values = starting_values or []
 
         if bottomless:
             maxlen = int(buffer_size / chunk_size)
         else:
             maxlen = None
 
         self.data = deque(starting_values, maxlen)
         self.worker = InputStreamChunker(source, self.data, buffer_size,
                                          chunk_size)
         if starting_values:
             self.worker.data_added.set()
         self.worker.start()
 
     ####################
     # Generator's methods
     ####################
 
     def __iter__(self):
         return self
 
     def next(self):
         while not len(self.data) and not self.worker.EOF.is_set():
             self.worker.data_added.clear()
             self.worker.data_added.wait(0.2)
         if len(self.data):
             self.worker.keep_reading.set()
             return bytes(self.data.popleft())
         elif self.worker.EOF.is_set():
             raise StopIteration
 
     def throw(self, exc_type, value=None, traceback=None):
         if not self.worker.EOF.is_set():
             raise exc_type(value)
 
     def start(self):
         self.worker.start()
 
     def stop(self):
         self.worker.stop()
 
     def close(self):
         try:
             self.worker.stop()
             self.throw(GeneratorExit)
         except (GeneratorExit, StopIteration):
             pass
 
     def __del__(self):
         self.close()
 
     ####################
     # Threaded reader's infrastructure.
     ####################
     @property
     def input(self):
         return self.worker.w
 
     @property
     def data_added_event(self):
         return self.worker.data_added
 
     @property
     def data_added(self):
         return self.worker.data_added.is_set()
 
     @property
     def reading_paused(self):
         return not self.worker.keep_reading.is_set()
 
     @property
     def done_reading_event(self):
         """
         Done_reding does not mean that the iterator's buffer is empty.
         Iterator might have done reading from underlying source, but the read
         chunks might still be available for serving through .next() method.
 
         :returns: An Event class instance.
         """
         return self.worker.EOF
 
     @property
     def done_reading(self):
         """
         Done_reding does not mean that the iterator's buffer is empty.
         Iterator might have done reading from underlying source, but the read
         chunks might still be available for serving through .next() method.
 
         :returns: An Bool value.
         """
         return self.worker.EOF.is_set()
 
     @property
     def length(self):
         """
         returns int.
 
         This is the lenght of the que of chunks, not the length of
         the combined contents in those chunks.
 
         __len__() cannot be meaningfully implemented because this
         reader is just flying throuh a bottomless pit content and
         can only know the lenght of what it already saw.
 
         If __len__() on WSGI server per PEP 3333 returns a value,
         the responce's length will be set to that. In order not to
         confuse WSGI PEP3333 servers, we will not implement __len__
         at all.
         """
         return len(self.data)
 
     def prepend(self, x):
         self.data.appendleft(x)
 
     def append(self, x):
         self.data.append(x)
 
     def extend(self, o):
         self.data.extend(o)
 
     def __getitem__(self, i):
         return self.data[i]
 
 
 class SubprocessIOChunker(object):
     """
     Processor class wrapping handling of subprocess IO.
 
     .. important::
 
        Watch out for the method `__del__` on this class. If this object
        is deleted, it will kill the subprocess, so avoid to
        return the `output` attribute or usage of it like in the following
        example::
 
           # `args` expected to run a program that produces a lot of output
           output = ''.join(SubprocessIOChunker(
              args, shell=False, inputstream=inputstream, env=environ).output)
 
           # `output` will not contain all the data, because the __del__ method
           # has already killed the subprocess in this case before all output
           # has been consumed.
 
 
 
     In a way, this is a "communicate()" replacement with a twist.
 
     - We are multithreaded. Writing in and reading out, err are all sep threads.
     - We support concurrent (in and out) stream processing.
     - The output is not a stream. It's a queue of read string (bytes, not unicode)
       chunks. The object behaves as an iterable. You can "for chunk in obj:" us.
     - We are non-blocking in more respects than communicate()
       (reading from subprocess out pauses when internal buffer is full, but
        does not block the parent calling code. On the flip side, reading from
        slow-yielding subprocess may block the iteration until data shows up. This
        does not block the parallel inpipe reading occurring parallel thread.)
 
     The purpose of the object is to allow us to wrap subprocess interactions into
     and interable that can be passed to a WSGI server as the application's return
     value. Because of stream-processing-ability, WSGI does not have to read ALL
     of the subprocess's output and buffer it, before handing it to WSGI server for
     HTTP response. Instead, the class initializer reads just a bit of the stream
     to figure out if error ocurred or likely to occur and if not, just hands the
     further iteration over subprocess output to the server for completion of HTTP
     response.
 
     The real or perceived subprocess error is trapped and raised as one of
     EnvironmentError family of exceptions
 
     Example usage:
     #    try:
     #        answer = SubprocessIOChunker(
     #            cmd,
     #            input,
     #            buffer_size = 65536,
     #            chunk_size = 4096
     #            )
     #    except (EnvironmentError) as e:
     #        print str(e)
     #        raise e
     #
     #    return answer
 
 
     """
 
     # TODO: johbo: This is used to make sure that the open end of the PIPE
     # is closed in the end. It would be way better to wrap this into an
     # object, so that it is closed automatically once it is consumed or
     # something similar.
     _close_input_fd = None
 
     _closed = False
 
     def __init__(self, cmd, inputstream=None, buffer_size=65536,
                  chunk_size=4096, starting_values=None, fail_on_stderr=True,
                  fail_on_return_code=True, **kwargs):
         """
         Initializes SubprocessIOChunker
 
         :param cmd: A Subprocess.Popen style "cmd". Can be string or array of strings
         :param inputstream: (Default: None) A file-like, string, or file pointer.
         :param buffer_size: (Default: 65536) A size of total buffer per stream in bytes.
         :param chunk_size: (Default: 4096) A max size of a chunk. Actual chunk may be smaller.
         :param starting_values: (Default: []) An array of strings to put in front of output que.
         :param fail_on_stderr: (Default: True) Whether to raise an exception in
                                case something is written to stderr.
         :param fail_on_return_code: (Default: True) Whether to raise an
                                     exception if the return code is not 0.
         """
 
         starting_values = starting_values or []
         if inputstream:
             input_streamer = StreamFeeder(inputstream)
             input_streamer.start()
             inputstream = input_streamer.output
             self._close_input_fd = inputstream
 
         self._fail_on_stderr = fail_on_stderr
         self._fail_on_return_code = fail_on_return_code
 
         _shell = kwargs.get('shell', True)
         kwargs['shell'] = _shell
 
         _p = subprocess.Popen(cmd, bufsize=-1,
                               stdin=inputstream,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               **kwargs)
 
         bg_out = BufferedGenerator(_p.stdout, buffer_size, chunk_size,
                                    starting_values)
         bg_err = BufferedGenerator(_p.stderr, 16000, 1, bottomless=True)
 
         while not bg_out.done_reading and not bg_out.reading_paused and not bg_err.length:
             # doing this until we reach either end of file, or end of buffer.
             bg_out.data_added_event.wait(1)
             bg_out.data_added_event.clear()
 
         # at this point it's still ambiguous if we are done reading or just full buffer.
         # Either way, if error (returned by ended process, or implied based on
         # presence of stuff in stderr output) we error out.
         # Else, we are happy.
         _returncode = _p.poll()
 
         if ((_returncode and fail_on_return_code) or
                 (fail_on_stderr and _returncode is None and bg_err.length)):
             try:
                 _p.terminate()
             except Exception:
                 pass
             bg_out.stop()
             bg_err.stop()
             if fail_on_stderr:
                 err = ''.join(bg_err)
                 raise EnvironmentError(
                     "Subprocess exited due to an error:\n" + err)
             if _returncode and fail_on_return_code:
                 err = ''.join(bg_err)
                 if not err:
                     # maybe get empty stderr, try stdout instead
                     # in many cases git reports the errors on stdout too
                     err = ''.join(bg_out)
                 raise EnvironmentError(
                     "Subprocess exited with non 0 ret code:%s: stderr:%s" % (
                         _returncode, err))
 
         self.process = _p
         self.output = bg_out
         self.error = bg_err
 
     def __iter__(self):
         return self
 
     def next(self):
         # Note: mikhail: We need to be sure that we are checking the return
         # code after the stdout stream is closed. Some processes, e.g. git
         # are doing some magic in between closing stdout and terminating the
         # process and, as a result,  we are not getting return code on "slow"
         # systems.
         result = None
         stop_iteration = None
         try:
             result = self.output.next()
         except StopIteration as e:
             stop_iteration = e
 
         if self.process.poll() and self._fail_on_return_code:
             err = '%s' % ''.join(self.error)
             raise EnvironmentError(
                 "Subprocess exited due to an error:\n" + err)
 
         if stop_iteration:
             raise stop_iteration
         return result
 
     def throw(self, type, value=None, traceback=None):
         if self.output.length or not self.output.done_reading:
             raise type(value)
 
     def close(self):
         if self._closed:
             return
         self._closed = True
         try:
             self.process.terminate()
         except:
             pass
         if self._close_input_fd:
             os.close(self._close_input_fd)
         try:
             self.output.close()
         except:
             pass
         try:
             self.error.close()
         except:
             pass
 
     def __del__(self):
         self.close()
+
+
+def run_command(arguments, env=None):
+    """
+    Run the specified command and return the stdout.
+
+    :param arguments: sequence of program arguments (including the program name)
+    :type arguments: list[str]
+    """
+
+    cmd = arguments
+    log.debug('Running subprocessio command %s', cmd)
+    try:
+        _opts = {'shell': False, 'fail_on_stderr': False}
+        if env:
+            _opts.update({'env': env})
+        p = SubprocessIOChunker(cmd, **_opts)
+        stdout = ''.join(p)
+        stderr = ''.join(''.join(p.error))
+    except (EnvironmentError, OSError) as err:
+        cmd = ' '.join(cmd)  # human friendly CMD
+        tb_err = ("Couldn't run subprocessio command (%s).\n"
+                  "Original error was:%s\n" % (cmd, err))
+        log.exception(tb_err)
+        raise Exception(tb_err)
+
+    return stdout, stderr

vcsserver/svn.py

@@ -1,679 +1,677 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2018 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 from __future__ import absolute_import
 
 import os
 from urllib2 import URLError
 import logging
 import posixpath as vcspath
 import StringIO
-import subprocess
 import urllib
 import traceback
 
 import svn.client
 import svn.core
 import svn.delta
 import svn.diff
 import svn.fs
 import svn.repos
 
-from vcsserver import svn_diff
-from vcsserver import exceptions
+from vcsserver import svn_diff, exceptions, subprocessio
 from vcsserver.base import RepoFactory, raise_from_original
 
-
 log = logging.getLogger(__name__)
 
 
 # Set of svn compatible version flags.
 # Compare with subversion/svnadmin/svnadmin.c
 svn_compatible_versions = set([
     'pre-1.4-compatible',
     'pre-1.5-compatible',
     'pre-1.6-compatible',
     'pre-1.8-compatible',
     'pre-1.9-compatible',
 ])
 
 svn_compatible_versions_map = {
     'pre-1.4-compatible': '1.3',
     'pre-1.5-compatible': '1.4',
     'pre-1.6-compatible': '1.5',
     'pre-1.8-compatible': '1.7',
     'pre-1.9-compatible': '1.8',
 }
 
 
 def reraise_safe_exceptions(func):
     """Decorator for converting svn exceptions to something neutral."""
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except Exception as e:
             if not hasattr(e, '_vcs_kind'):
                 log.exception("Unhandled exception in hg remote call")
                 raise_from_original(exceptions.UnhandledException)
             raise
     return wrapper
 
 
 class SubversionFactory(RepoFactory):
 
     def _create_repo(self, wire, create, compatible_version):
         path = svn.core.svn_path_canonicalize(wire['path'])
         if create:
             fs_config = {'compatible-version': '1.9'}
             if compatible_version:
                 if compatible_version not in svn_compatible_versions:
                     raise Exception('Unknown SVN compatible version "{}"'
                                     .format(compatible_version))
                 fs_config['compatible-version'] = \
                     svn_compatible_versions_map[compatible_version]
 
             log.debug('Create SVN repo with config "%s"', fs_config)
             repo = svn.repos.create(path, "", "", None, fs_config)
         else:
             repo = svn.repos.open(path)
 
         log.debug('Got SVN object: %s', repo)
         return repo
 
     def repo(self, wire, create=False, compatible_version=None):
         def create_new_repo():
             return self._create_repo(wire, create, compatible_version)
 
         return self._repo(wire, create_new_repo)
 
 
 NODE_TYPE_MAPPING = {
     svn.core.svn_node_file: 'file',
     svn.core.svn_node_dir: 'dir',
 }
 
 
 class SvnRemote(object):
 
     def __init__(self, factory, hg_factory=None):
         self._factory = factory
         # TODO: Remove once we do not use internal Mercurial objects anymore
         # for subversion
         self._hg_factory = hg_factory
 
     @reraise_safe_exceptions
     def discover_svn_version(self):
         try:
             import svn.core
             svn_ver = svn.core.SVN_VERSION
         except ImportError:
             svn_ver = None
         return svn_ver
 
     def check_url(self, url, config_items):
         # this can throw exception if not installed, but we detect this
         from hgsubversion import svnrepo
 
         baseui = self._hg_factory._create_config(config_items)
         # uuid function get's only valid UUID from proper repo, else
         # throws exception
         try:
             svnrepo.svnremoterepo(baseui, url).svn.uuid
         except Exception:
             tb = traceback.format_exc()
             log.debug("Invalid Subversion url: `%s`, tb: %s", url, tb)
             raise URLError(
                 '"%s" is not a valid Subversion source url.' % (url, ))
         return True
 
     def is_path_valid_repository(self, wire, path):
 
         # NOTE(marcink):  short circuit the check for SVN repo
         # the repos.open might be expensive to check, but we have one cheap
         # pre condition that we can use, to check for 'format' file
 
         if not os.path.isfile(os.path.join(path, 'format')):
             return False
 
         try:
             svn.repos.open(path)
         except svn.core.SubversionException:
             tb = traceback.format_exc()
             log.debug("Invalid Subversion path `%s`, tb: %s", path, tb)
             return False
         return True
 
     @reraise_safe_exceptions
     def verify(self, wire,):
         repo_path = wire['path']
         if not self.is_path_valid_repository(wire, repo_path):
             raise Exception(
                 "Path %s is not a valid Subversion repository." % repo_path)
 
-        load = subprocess.Popen(
-            ['svnadmin', 'info', repo_path],
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        return ''.join(load.stdout)
+        cmd = ['svnadmin', 'info', repo_path]
+        stdout, stderr = subprocessio.run_command(cmd)
+        return stdout
 
     def lookup(self, wire, revision):
         if revision not in [-1, None, 'HEAD']:
             raise NotImplementedError
         repo = self._factory.repo(wire)
         fs_ptr = svn.repos.fs(repo)
         head = svn.fs.youngest_rev(fs_ptr)
         return head
 
     def lookup_interval(self, wire, start_ts, end_ts):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         start_rev = None
         end_rev = None
         if start_ts:
             start_ts_svn = apr_time_t(start_ts)
             start_rev = svn.repos.dated_revision(repo, start_ts_svn) + 1
         else:
             start_rev = 1
         if end_ts:
             end_ts_svn = apr_time_t(end_ts)
             end_rev = svn.repos.dated_revision(repo, end_ts_svn)
         else:
             end_rev = svn.fs.youngest_rev(fsobj)
         return start_rev, end_rev
 
     def revision_properties(self, wire, revision):
         repo = self._factory.repo(wire)
         fs_ptr = svn.repos.fs(repo)
         return svn.fs.revision_proplist(fs_ptr, revision)
 
     def revision_changes(self, wire, revision):
 
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         rev_root = svn.fs.revision_root(fsobj, revision)
 
         editor = svn.repos.ChangeCollector(fsobj, rev_root)
         editor_ptr, editor_baton = svn.delta.make_editor(editor)
         base_dir = ""
         send_deltas = False
         svn.repos.replay2(
             rev_root, base_dir, svn.core.SVN_INVALID_REVNUM, send_deltas,
             editor_ptr, editor_baton, None)
 
         added = []
         changed = []
         removed = []
 
         # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
         for path, change in editor.changes.iteritems():
             # TODO: Decide what to do with directory nodes. Subversion can add
             # empty directories.
 
             if change.item_kind == svn.core.svn_node_dir:
                 continue
             if change.action in [svn.repos.CHANGE_ACTION_ADD]:
                 added.append(path)
             elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
                                    svn.repos.CHANGE_ACTION_REPLACE]:
                 changed.append(path)
             elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
                 removed.append(path)
             else:
                 raise NotImplementedError(
                     "Action %s not supported on path %s" % (
                         change.action, path))
 
         changes = {
             'added': added,
             'changed': changed,
             'removed': removed,
         }
         return changes
 
     def node_history(self, wire, path, revision, limit):
         cross_copies = False
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         rev_root = svn.fs.revision_root(fsobj, revision)
 
         history_revisions = []
         history = svn.fs.node_history(rev_root, path)
         history = svn.fs.history_prev(history, cross_copies)
         while history:
             __, node_revision = svn.fs.history_location(history)
             history_revisions.append(node_revision)
             if limit and len(history_revisions) >= limit:
                 break
             history = svn.fs.history_prev(history, cross_copies)
         return history_revisions
 
     def node_properties(self, wire, path, revision):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         rev_root = svn.fs.revision_root(fsobj, revision)
         return svn.fs.node_proplist(rev_root, path)
 
     def file_annotate(self, wire, path, revision):
         abs_path = 'file://' + urllib.pathname2url(
             vcspath.join(wire['path'], path))
         file_uri = svn.core.svn_path_canonicalize(abs_path)
 
         start_rev = svn_opt_revision_value_t(0)
         peg_rev = svn_opt_revision_value_t(revision)
         end_rev = peg_rev
 
         annotations = []
 
         def receiver(line_no, revision, author, date, line, pool):
             annotations.append((line_no, revision, line))
 
         # TODO: Cannot use blame5, missing typemap function in the swig code
         try:
             svn.client.blame2(
                 file_uri, peg_rev, start_rev, end_rev,
                 receiver, svn.client.create_context())
         except svn.core.SubversionException as exc:
             log.exception("Error during blame operation.")
             raise Exception(
                 "Blame not supported or file does not exist at path %s. "
                 "Error %s." % (path, exc))
 
         return annotations
 
     def get_node_type(self, wire, path, rev=None):
         repo = self._factory.repo(wire)
         fs_ptr = svn.repos.fs(repo)
         if rev is None:
             rev = svn.fs.youngest_rev(fs_ptr)
         root = svn.fs.revision_root(fs_ptr, rev)
         node = svn.fs.check_path(root, path)
         return NODE_TYPE_MAPPING.get(node, None)
 
     def get_nodes(self, wire, path, revision=None):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         if revision is None:
             revision = svn.fs.youngest_rev(fsobj)
         root = svn.fs.revision_root(fsobj, revision)
         entries = svn.fs.dir_entries(root, path)
         result = []
         for entry_path, entry_info in entries.iteritems():
             result.append(
                 (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
         return result
 
     def get_file_content(self, wire, path, rev=None):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         if rev is None:
             rev = svn.fs.youngest_revision(fsobj)
         root = svn.fs.revision_root(fsobj, rev)
         content = svn.core.Stream(svn.fs.file_contents(root, path))
         return content.read()
 
     def get_file_size(self, wire, path, revision=None):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         if revision is None:
             revision = svn.fs.youngest_revision(fsobj)
         root = svn.fs.revision_root(fsobj, revision)
         size = svn.fs.file_length(root, path)
         return size
 
     def create_repository(self, wire, compatible_version=None):
         log.info('Creating Subversion repository in path "%s"', wire['path'])
         self._factory.repo(wire, create=True,
                            compatible_version=compatible_version)
 
     def import_remote_repository(self, wire, src_url):
         repo_path = wire['path']
         if not self.is_path_valid_repository(wire, repo_path):
             raise Exception(
                 "Path %s is not a valid Subversion repository." % repo_path)
+
         # TODO: johbo: URL checks ?
+        import subprocess
         rdump = subprocess.Popen(
             ['svnrdump', 'dump', '--non-interactive', src_url],
             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         load = subprocess.Popen(
             ['svnadmin', 'load', repo_path], stdin=rdump.stdout)
 
         # TODO: johbo: This can be a very long operation, might be better
         # to track some kind of status and provide an api to check if the
         # import is done.
         rdump.wait()
         load.wait()
 
         if rdump.returncode != 0:
             errors = rdump.stderr.read()
             log.error('svnrdump dump failed: statuscode %s: message: %s',
                       rdump.returncode, errors)
             reason = 'UNKNOWN'
             if 'svnrdump: E230001:' in errors:
                 reason = 'INVALID_CERTIFICATE'
             raise Exception(
                 'Failed to dump the remote repository from %s.' % src_url,
                 reason)
         if load.returncode != 0:
             raise Exception(
                 'Failed to load the dump of remote repository from %s.' %
                 (src_url, ))
 
     def commit(self, wire, message, author, timestamp, updated, removed):
         assert isinstance(message, str)
         assert isinstance(author, str)
 
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
 
         rev = svn.fs.youngest_rev(fsobj)
         txn = svn.repos.fs_begin_txn_for_commit(repo, rev, author, message)
         txn_root = svn.fs.txn_root(txn)
 
         for node in updated:
             TxnNodeProcessor(node, txn_root).update()
         for node in removed:
             TxnNodeProcessor(node, txn_root).remove()
 
         commit_id = svn.repos.fs_commit_txn(repo, txn)
 
         if timestamp:
             apr_time = apr_time_t(timestamp)
             ts_formatted = svn.core.svn_time_to_cstring(apr_time)
             svn.fs.change_rev_prop(fsobj, commit_id, 'svn:date', ts_formatted)
 
         log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
         return commit_id
 
     def diff(self, wire, rev1, rev2, path1=None, path2=None,
              ignore_whitespace=False, context=3):
 
         wire.update(cache=False)
         repo = self._factory.repo(wire)
         diff_creator = SvnDiffer(
             repo, rev1, path1, rev2, path2, ignore_whitespace, context)
         try:
             return diff_creator.generate_diff()
         except svn.core.SubversionException as e:
             log.exception(
                 "Error during diff operation operation. "
                 "Path might not exist %s, %s" % (path1, path2))
             return ""
 
     @reraise_safe_exceptions
     def is_large_file(self, wire, path):
         return False
 
 
 class SvnDiffer(object):
     """
     Utility to create diffs based on difflib and the Subversion api
     """
 
     binary_content = False
 
     def __init__(
             self, repo, src_rev, src_path, tgt_rev, tgt_path,
             ignore_whitespace, context):
         self.repo = repo
         self.ignore_whitespace = ignore_whitespace
         self.context = context
 
         fsobj = svn.repos.fs(repo)
 
         self.tgt_rev = tgt_rev
         self.tgt_path = tgt_path or ''
         self.tgt_root = svn.fs.revision_root(fsobj, tgt_rev)
         self.tgt_kind = svn.fs.check_path(self.tgt_root, self.tgt_path)
 
         self.src_rev = src_rev
         self.src_path = src_path or self.tgt_path
         self.src_root = svn.fs.revision_root(fsobj, src_rev)
         self.src_kind = svn.fs.check_path(self.src_root, self.src_path)
 
         self._validate()
 
     def _validate(self):
         if (self.tgt_kind != svn.core.svn_node_none and
                 self.src_kind != svn.core.svn_node_none and
                 self.src_kind != self.tgt_kind):
             # TODO: johbo: proper error handling
             raise Exception(
                 "Source and target are not compatible for diff generation. "
                 "Source type: %s, target type: %s" %
                 (self.src_kind, self.tgt_kind))
 
     def generate_diff(self):
         buf = StringIO.StringIO()
         if self.tgt_kind == svn.core.svn_node_dir:
             self._generate_dir_diff(buf)
         else:
             self._generate_file_diff(buf)
         return buf.getvalue()
 
     def _generate_dir_diff(self, buf):
         editor = DiffChangeEditor()
         editor_ptr, editor_baton = svn.delta.make_editor(editor)
         svn.repos.dir_delta2(
             self.src_root,
             self.src_path,
             '',  # src_entry
             self.tgt_root,
             self.tgt_path,
             editor_ptr, editor_baton,
             authorization_callback_allow_all,
             False,  # text_deltas
             svn.core.svn_depth_infinity,  # depth
             False,  # entry_props
             False,  # ignore_ancestry
         )
 
         for path, __, change in sorted(editor.changes):
             self._generate_node_diff(
                 buf, change, path, self.tgt_path, path, self.src_path)
 
     def _generate_file_diff(self, buf):
         change = None
         if self.src_kind == svn.core.svn_node_none:
             change = "add"
         elif self.tgt_kind == svn.core.svn_node_none:
             change = "delete"
         tgt_base, tgt_path = vcspath.split(self.tgt_path)
         src_base, src_path = vcspath.split(self.src_path)
         self._generate_node_diff(
             buf, change, tgt_path, tgt_base, src_path, src_base)
 
     def _generate_node_diff(
             self, buf, change, tgt_path, tgt_base, src_path, src_base):
 
         if self.src_rev == self.tgt_rev and tgt_base == src_base:
             # makes consistent behaviour with git/hg to return empty diff if
             # we compare same revisions
             return
 
         tgt_full_path = vcspath.join(tgt_base, tgt_path)
         src_full_path = vcspath.join(src_base, src_path)
 
         self.binary_content = False
         mime_type = self._get_mime_type(tgt_full_path)
 
         if mime_type and not mime_type.startswith('text'):
             self.binary_content = True
             buf.write("=" * 67 + '\n')
             buf.write("Cannot display: file marked as a binary type.\n")
             buf.write("svn:mime-type = %s\n" % mime_type)
         buf.write("Index: %s\n" % (tgt_path, ))
         buf.write("=" * 67 + '\n')
         buf.write("diff --git a/%(tgt_path)s b/%(tgt_path)s\n" % {
             'tgt_path': tgt_path})
 
         if change == 'add':
             # TODO: johbo: SVN is missing a zero here compared to git
             buf.write("new file mode 10644\n")
 
             #TODO(marcink): intro to binary detection of svn patches
             # if self.binary_content:
             #     buf.write('GIT binary patch\n')
 
             buf.write("--- /dev/null\t(revision 0)\n")
             src_lines = []
         else:
             if change == 'delete':
                 buf.write("deleted file mode 10644\n")
 
             #TODO(marcink): intro to binary detection of svn patches
             # if self.binary_content:
             #     buf.write('GIT binary patch\n')
 
             buf.write("--- a/%s\t(revision %s)\n" % (
                 src_path, self.src_rev))
             src_lines = self._svn_readlines(self.src_root, src_full_path)
 
         if change == 'delete':
             buf.write("+++ /dev/null\t(revision %s)\n" % (self.tgt_rev, ))
             tgt_lines = []
         else:
             buf.write("+++ b/%s\t(revision %s)\n" % (
                 tgt_path, self.tgt_rev))
             tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
 
         if not self.binary_content:
             udiff = svn_diff.unified_diff(
                 src_lines, tgt_lines, context=self.context,
                 ignore_blank_lines=self.ignore_whitespace,
                 ignore_case=False,
                 ignore_space_changes=self.ignore_whitespace)
             buf.writelines(udiff)
 
     def _get_mime_type(self, path):
         try:
             mime_type = svn.fs.node_prop(
                 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
         except svn.core.SubversionException:
             mime_type = svn.fs.node_prop(
                 self.src_root, path, svn.core.SVN_PROP_MIME_TYPE)
         return mime_type
 
     def _svn_readlines(self, fs_root, node_path):
         if self.binary_content:
             return []
         node_kind = svn.fs.check_path(fs_root, node_path)
         if node_kind not in (
                 svn.core.svn_node_file, svn.core.svn_node_symlink):
             return []
         content = svn.core.Stream(
             svn.fs.file_contents(fs_root, node_path)).read()
         return content.splitlines(True)
 
 
 class DiffChangeEditor(svn.delta.Editor):
     """
     Records changes between two given revisions
     """
 
     def __init__(self):
         self.changes = []
 
     def delete_entry(self, path, revision, parent_baton, pool=None):
         self.changes.append((path, None, 'delete'))
 
     def add_file(
             self, path, parent_baton, copyfrom_path, copyfrom_revision,
             file_pool=None):
         self.changes.append((path, 'file', 'add'))
 
     def open_file(self, path, parent_baton, base_revision, file_pool=None):
         self.changes.append((path, 'file', 'change'))
 
 
 def authorization_callback_allow_all(root, path, pool):
     return True
 
 
 class TxnNodeProcessor(object):
     """
     Utility to process the change of one node within a transaction root.
 
     It encapsulates the knowledge of how to add, update or remove
     a node for a given transaction root. The purpose is to support the method
     `SvnRemote.commit`.
     """
 
     def __init__(self, node, txn_root):
         assert isinstance(node['path'], str)
 
         self.node = node
         self.txn_root = txn_root
 
     def update(self):
         self._ensure_parent_dirs()
         self._add_file_if_node_does_not_exist()
         self._update_file_content()
         self._update_file_properties()
 
     def remove(self):
         svn.fs.delete(self.txn_root, self.node['path'])
         # TODO: Clean up directory if empty
 
     def _ensure_parent_dirs(self):
         curdir = vcspath.dirname(self.node['path'])
         dirs_to_create = []
         while not self._svn_path_exists(curdir):
             dirs_to_create.append(curdir)
             curdir = vcspath.dirname(curdir)
 
         for curdir in reversed(dirs_to_create):
             log.debug('Creating missing directory "%s"', curdir)
             svn.fs.make_dir(self.txn_root, curdir)
 
     def _svn_path_exists(self, path):
         path_status = svn.fs.check_path(self.txn_root, path)
         return path_status != svn.core.svn_node_none
 
     def _add_file_if_node_does_not_exist(self):
         kind = svn.fs.check_path(self.txn_root, self.node['path'])
         if kind == svn.core.svn_node_none:
             svn.fs.make_file(self.txn_root, self.node['path'])
 
     def _update_file_content(self):
         assert isinstance(self.node['content'], str)
         handler, baton = svn.fs.apply_textdelta(
             self.txn_root, self.node['path'], None, None)
         svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
 
     def _update_file_properties(self):
         properties = self.node.get('properties', {})
         for key, value in properties.iteritems():
             svn.fs.change_node_prop(
                 self.txn_root, self.node['path'], key, value)
 
 
 def apr_time_t(timestamp):
     """
     Convert a Python timestamp into APR timestamp type apr_time_t
     """
     return timestamp * 1E6
 
 
 def svn_opt_revision_value_t(num):
     """
     Put `num` into a `svn_opt_revision_value_t` structure.
     """
     value = svn.core.svn_opt_revision_value_t()
     value.number = num
     revision = svn.core.svn_opt_revision_t()
     revision.kind = svn.core.svn_opt_revision_number
     revision.value = value
     return revision