py3: add safe_str where we really need it to get a str - probably from bytes

kallithea/config/app_cfg.py

@@ -40,7 +40,7 b' from kallithea.lib.middleware.simplegit '
 from kallithea.lib.middleware.simplehg import SimpleHg
 from kallithea.lib.middleware.wrapper import RequestWrapper
 from kallithea.lib.utils import check_git_version, load_rcextensions, make_ui, set_app_settings, set_indexer_config, set_vcs_config
-from kallithea.lib.utils2 import str2bool
+from kallithea.lib.utils2 import safe_str, str2bool
 
 
 log = logging.getLogger(__name__)
@@ -163,7 +163,7 b' def setup_configuration(app):'
 
     load_rcextensions(root_path=config['here'])
 
-    repos_path = make_ui().configitems(b'paths')[0][1]
+    repos_path = safe_str(make_ui().configitems(b'paths')[0][1])
     config['base_path'] = repos_path
     set_app_settings(config)
 

kallithea/controllers/home.py

@@ -37,6 +37,7 b' from webob.exc import HTTPBadRequest'
 from kallithea.lib import helpers as h
 from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired
 from kallithea.lib.base import BaseController, jsonify, render
+from kallithea.lib.utils2 import safe_str
 from kallithea.model.db import RepoGroup, Repository, User, UserGroup
 from kallithea.model.repo import RepoModel
 from kallithea.model.scm import UserGroupList
@@ -116,25 +117,25 b' class HomeController(BaseController):'
         if _branches:
             res.append({
                 'text': _('Branch'),
-                'children': [{'id': rev, 'text': name, 'type': 'branch'} for name, rev in _branches]
+                'children': [{'id': safe_str(rev), 'text': safe_str(name), 'type': 'branch'} for name, rev in _branches]
             })
         _closed_branches = repo.closed_branches.items()
         if _closed_branches:
             res.append({
                 'text': _('Closed Branches'),
-                'children': [{'id': rev, 'text': name, 'type': 'closed-branch'} for name, rev in _closed_branches]
+                'children': [{'id': safe_str(rev), 'text': safe_str(name), 'type': 'closed-branch'} for name, rev in _closed_branches]
             })
         _tags = repo.tags.items()
         if _tags:
             res.append({
                 'text': _('Tag'),
-                'children': [{'id': rev, 'text': name, 'type': 'tag'} for name, rev in _tags]
+                'children': [{'id': safe_str(rev), 'text': safe_str(name), 'type': 'tag'} for name, rev in _tags]
             })
         _bookmarks = repo.bookmarks.items()
         if _bookmarks:
             res.append({
                 'text': _('Bookmark'),
-                'children': [{'id': rev, 'text': name, 'type': 'book'} for name, rev in _bookmarks]
+                'children': [{'id': safe_str(rev), 'text': safe_str(name), 'type': 'book'} for name, rev in _bookmarks]
             })
         data = {
             'more': False,

kallithea/lib/base.py

@@ -172,7 +172,7 b' class BasicAuth(paste.auth.basic.AuthBas'
         (authmeth, auth) = authorization.split(' ', 1)
         if 'basic' != authmeth.lower():
             return self.build_authentication(environ)
-        auth = base64.b64decode(auth.strip())
+        auth = safe_str(base64.b64decode(auth.strip()))
         _parts = auth.split(':', 1)
         if len(_parts) == 2:
             username, password = _parts

kallithea/lib/diffs.py

@@ -538,10 +538,10 b' def _get_header(vcs, diff_chunk):'
         match = _hg_header_re.match(diff_chunk)
     if match is None:
         raise Exception('diff not recognized as valid %s diff' % vcs)
-    meta_info = match.groupdict()
+    meta_info = {k: None if v is None else safe_str(v) for k, v in match.groupdict().items()}
     rest = diff_chunk[match.end():]
     if rest and _header_next_check.match(rest):
-        raise Exception('cannot parse %s diff header: %r followed by %r' % (vcs, diff_chunk[:match.end()], rest[:1000]))
+        raise Exception('cannot parse %s diff header: %r followed by %r' % (vcs, safe_str(bytes(diff_chunk[:match.end()])), safe_str(bytes(rest[:1000]))))
     diff_lines = (_escaper(m.group(0)) for m in re.finditer(br'.*\n|.+$', rest)) # don't split on \r as str.splitlines do
     return meta_info, diff_lines
 

kallithea/lib/hooks.py

@@ -34,7 +34,7 b' import mercurial.scmutil'
 from kallithea.lib import helpers as h
 from kallithea.lib.exceptions import UserCreationError
 from kallithea.lib.utils import action_logger, make_ui
-from kallithea.lib.utils2 import HookEnvironmentError, ascii_str, get_hook_environment, safe_bytes
+from kallithea.lib.utils2 import HookEnvironmentError, ascii_str, get_hook_environment, safe_bytes, safe_str
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.model.db import Repository, User
 
@@ -67,7 +67,7 b' def _get_scm_size(alias, root_path):'
 
 def repo_size(ui, repo, hooktype=None, **kwargs):
     """Show size of Mercurial repository, to be called after push."""
-    size_hg_f, size_root_f, size_total_f = _get_scm_size('.hg', repo.root)
+    size_hg_f, size_root_f, size_total_f = _get_scm_size('.hg', safe_str(repo.root))
 
     last_cs = repo[len(repo) - 1]
 

kallithea/lib/utils.py

@@ -40,7 +40,7 b' from tg.i18n import ugettext as _'
 
 import kallithea.config.conf
 from kallithea.lib.exceptions import HgsubversionImportError
-from kallithea.lib.utils2 import ascii_bytes, aslist, get_current_authuser, safe_bytes
+from kallithea.lib.utils2 import ascii_bytes, aslist, get_current_authuser, safe_bytes, safe_str
 from kallithea.lib.vcs.backends.git.repository import GitRepository
 from kallithea.lib.vcs.backends.hg.repository import MercurialRepository
 from kallithea.lib.vcs.conf import settings
@@ -586,13 +586,13 b' def check_git_version():'
         return None
 
     if stderr:
-        log.warning('Error/stderr from "%s --version":\n%s', settings.GIT_EXECUTABLE_PATH, stderr)
+        log.warning('Error/stderr from "%s --version":\n%s', settings.GIT_EXECUTABLE_PATH, safe_str(stderr))
 
     if not stdout:
         log.warning('No working git executable found - check "git_path" in the ini file.')
         return None
 
-    output = stdout.strip()
+    output = safe_str(stdout).strip()
     m = re.search(r"\d+.\d+.\d+", output)
     if m:
         ver = StrictVersion(m.group(0))

kallithea/lib/vcs/backends/git/changeset.py

@@ -96,7 +96,7 b' class GitChangeset(BaseChangeset):'
     @LazyProperty
     def branches(self):
         heads = self.repository._heads(reverse=True)
-        return [b for b in heads if heads[b] == self._commit.id] # FIXME: Inefficient ... and returning None!
+        return [safe_str(b) for b in heads if heads[b] == self._commit.id] # FIXME: Inefficient ... and returning None!
 
     def _fix_path(self, path):
         """
@@ -122,10 +122,9 b' class GitChangeset(BaseChangeset):'
 
             # initially extract things from root dir
             for item, stat, id in tree.items():
+                name = safe_str(item)
                 if curdir:
-                    name = '/'.join((curdir, item))
-                else:
-                    name = item
+                    name = '/'.join((curdir, name))
                 self._paths[name] = id
                 self._stat_modes[name] = stat
 
@@ -136,7 +135,8 b' class GitChangeset(BaseChangeset):'
                     curdir = dir
                 dir_id = None
                 for item, stat, id in tree.items():
-                    if dir == item:
+                    name = safe_str(item)
+                    if dir == name:
                         dir_id = id
                 if dir_id:
                     # Update tree
@@ -148,10 +148,9 b' class GitChangeset(BaseChangeset):'
 
                 # cache all items from the given traversed tree
                 for item, stat, id in tree.items():
+                    name = safe_str(item)
                     if curdir:
-                        name = '/'.join((curdir, item))
-                    else:
-                        name = item
+                        name = '/'.join((curdir, name))
                     self._paths[name] = id
                     self._stat_modes[name] = stat
             if path not in self._paths:
@@ -404,10 +403,9 b' class GitChangeset(BaseChangeset):'
         filenodes = []
         als = self.repository.alias
         for name, stat, id in tree.items():
+            obj_path = safe_str(name)
             if path != '':
-                obj_path = '/'.join((path, name))
-            else:
-                obj_path = name
+                obj_path = '/'.join((path, obj_path))
             if objects.S_ISGITLINK(stat):
                 root_tree = self.repository._repo[self._tree_id]
                 cf = ConfigFile.from_file(BytesIO(self.repository._repo.get_object(root_tree[b'.gitmodules'][1]).data))
@@ -499,11 +497,11 b' class GitChangeset(BaseChangeset):'
             changes = _r.object_store.tree_changes(oid, _r[self._commit.id].tree)
             for (oldpath, newpath), (_, _), (_, _) in changes:
                 if newpath and oldpath:
-                    modified.add(newpath)
+                    modified.add(safe_str(newpath))
                 elif newpath and not oldpath:
-                    added.add(newpath)
+                    added.add(safe_str(newpath))
                 elif not newpath and oldpath:
-                    deleted.add(oldpath)
+                    deleted.add(safe_str(oldpath))
         return added, modified, deleted
 
     def _get_paths_for_status(self, status):

kallithea/lib/vcs/backends/git/repository.py

@@ -359,7 +359,7 b' class GitRepository(BaseRepository):'
         if not self.revisions:
             return {}
         sortkey = lambda ctx: ctx[0]
-        _branches = [(key, ascii_str(sha))
+        _branches = [(safe_str(key), ascii_str(sha))
                      for key, (sha, type_) in self._parsed_refs.items() if type_ == b'H']
         return OrderedDict(sorted(_branches, key=sortkey, reverse=False))
 
@@ -376,7 +376,7 b' class GitRepository(BaseRepository):'
             return {}
 
         sortkey = lambda ctx: ctx[0]
-        _tags = [(key, ascii_str(sha))
+        _tags = [(safe_str(key), ascii_str(sha))
                  for key, (sha, type_) in self._parsed_refs.items() if type_ == b'T']
         return OrderedDict(sorted(_tags, key=sortkey, reverse=True))
 
@@ -418,7 +418,7 b' class GitRepository(BaseRepository):'
         if name not in self.tags:
             raise TagDoesNotExistError("Tag %s does not exist" % name)
         # self._repo.refs is a DiskRefsContainer, and .path gives the full absolute path of '.git'
-        tagpath = os.path.join(self._repo.refs.path, 'refs', 'tags', name)
+        tagpath = os.path.join(safe_str(self._repo.refs.path), 'refs', 'tags', name)
         try:
             os.remove(tagpath)
             self._parsed_refs = self._get_parsed_refs()
@@ -712,9 +712,10 b' class GitRepository(BaseRepository):'
 
         for config in gen_configs():
             try:
-                return config.get(section, name)
+                value = config.get(section, name)
             except KeyError:
                 continue
+            return None if value is None else safe_str(value)
         return None
 
     def get_user_name(self, config_file=None):

kallithea/lib/vcs/backends/git/workdir.py

@@ -1,6 +1,6 b''
 import re
 
-from kallithea.lib.utils2 import ascii_str
+from kallithea.lib.utils2 import ascii_str, safe_str
 from kallithea.lib.vcs.backends.base import BaseWorkdir
 from kallithea.lib.vcs.exceptions import BranchDoesNotExistError, RepositoryError
 
@@ -10,7 +10,7 b' class GitWorkdir(BaseWorkdir):'
     def get_branch(self):
         headpath = self.repository._repo.refs.refpath(b'HEAD')
         try:
-            content = open(headpath).read()
+            content = safe_str(open(headpath, 'rb').read())
             match = re.match(r'^ref: refs/heads/(?P<branch>.+)\n$', content)
             if match:
                 return match.groupdict()['branch']

kallithea/lib/vcs/backends/hg/changeset.py

@@ -121,7 +121,7 b' class MercurialChangeset(BaseChangeset):'
 
     @LazyProperty
     def _file_paths(self):
-        return list(self._ctx)
+        return list(safe_str(f) for f in self._ctx)
 
     @LazyProperty
     def _dir_paths(self):
@@ -386,21 +386,21 b' class MercurialChangeset(BaseChangeset):'
         """
         Returns list of added ``FileNode`` objects.
         """
-        return AddedFileNodesGenerator([n for n in self.status.added], self)
+        return AddedFileNodesGenerator([safe_str(n) for n in self.status.added], self)
 
     @property
     def changed(self):
         """
         Returns list of modified ``FileNode`` objects.
         """
-        return ChangedFileNodesGenerator([n for n in self.status.modified], self)
+        return ChangedFileNodesGenerator([safe_str(n) for n in self.status.modified], self)
 
     @property
     def removed(self):
         """
         Returns list of removed ``FileNode`` objects.
         """
-        return RemovedFileNodesGenerator([n for n in self.status.removed], self)
+        return RemovedFileNodesGenerator([safe_str(n) for n in self.status.removed], self)
 
     @LazyProperty
     def extra(self):

kallithea/lib/vcs/backends/hg/inmemory.py

@@ -5,7 +5,7 b' import mercurial.node'
 
 from kallithea.lib.vcs.backends.base import BaseInMemoryChangeset
 from kallithea.lib.vcs.exceptions import RepositoryError
-from kallithea.lib.vcs.utils import ascii_str, safe_bytes
+from kallithea.lib.vcs.utils import ascii_str, safe_bytes, safe_str
 
 
 class MercurialInMemoryChangeset(BaseInMemoryChangeset):
@@ -45,7 +45,7 b' class MercurialInMemoryChangeset(BaseInM'
             Callback from Mercurial, returning ctx to commit for the given
             path.
             """
-            path = bytes_path  # will be different for py3
+            path = safe_str(bytes_path)
 
             # check if this path is removed
             if path in (node.path for node in self.removed):

kallithea/lib/vcs/backends/hg/repository.py

@@ -480,7 +480,7 b' class MercurialRepository(BaseRepository'
             if name in allowed or self._repo.ui.configbool(b"web",
                                                            b"allow" + name,
                                                            untrusted=True):
-                yield {"type": name, "extension": ext, "node": archive_name}
+                yield {"type": safe_str(name), "extension": ext, "node": archive_name}
 
     def _get_url(self, url):
         """
@@ -589,7 +589,8 b' class MercurialRepository(BaseRepository'
             config = mercurial.ui.ui()
             for path in config_file:
                 config.readconfig(safe_bytes(path))
-        return config.config(safe_bytes(section), safe_bytes(name))
+        value = config.config(safe_bytes(section), safe_bytes(name))
+        return value if value is None else safe_str(value)
 
     def get_user_name(self, config_file=None):
         """

kallithea/lib/vcs/backends/hg/workdir.py

@@ -2,13 +2,13 b' import mercurial.merge'
 
 from kallithea.lib.vcs.backends.base import BaseWorkdir
 from kallithea.lib.vcs.exceptions import BranchDoesNotExistError
-from kallithea.lib.vcs.utils import ascii_bytes, ascii_str
+from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, safe_str
 
 
 class MercurialWorkdir(BaseWorkdir):
 
     def get_branch(self):
-        return self.repository._repo.dirstate.branch()
+        return safe_str(self.repository._repo.dirstate.branch())
 
     def get_changeset(self):
         wk_dir_id = ascii_str(self.repository._repo[None].parents()[0].hex())