tests: use gunicorn for testing. This is close to production testing

and actually will reproduce any chunked encoding problems as we hit on production
servers.

rhodecode/tests/other/vcs_operations/conftest.py

@@ -171,7 +171,7 b' def rc_web_server('
 
     host_url = 'http://' + get_host_url(rc_web_server_config)
     assert_no_running_instance(host_url)
-    command = ['pserve', rc_web_server_config]
+    command = ['gunicorn', '--worker-class', 'gevent', '--paste', rc_web_server_config]
 
     print('Starting rhodecode server: {}'.format(host_url))
     print('Command: {}'.format(command))

rhodecode/tests/pylons_plugin.py

@@ -22,6 +22,8 b' import os'
 import json
 import platform
 import socket
+import tempfile
+
 import subprocess32
 import time
 from urllib2 import urlopen, URLError
@@ -34,6 +36,9 b' import pyramid.paster'
 from rhodecode.lib.pyramid_utils import get_app_config
 from rhodecode.tests.fixture import TestINI
 import rhodecode
+from rhodecode.tests.other.vcs_operations.conftest import get_host_url, get_port
+
+VCSSERVER_LOG = os.path.join(tempfile.gettempdir(), 'rc-vcsserver.log')
 
 
 def _parse_json(value):
@@ -204,10 +209,11 b' class HttpVCSServer(VCSServer):'
     Represents a running VCSServer instance.
     """
     def __init__(self, config_file):
+        self.config_file = config_file
         config_data = configobj.ConfigObj(config_file)
         self._config = config_data['server:main']
 
-        args = ['pserve', config_file]
+        args = ['gunicorn', '--workers', '1', '--paste', config_file]
         self._args = args
 
     @property
@@ -216,7 +222,21 b' class HttpVCSServer(VCSServer):'
         return template.format(**self._config)
 
     def start(self):
-        self.process = subprocess32.Popen(self._args)
+        env = os.environ.copy()
+        host_url = 'http://' + get_host_url(self.config_file)
+
+        rc_log = list(VCSSERVER_LOG.partition('.log'))
+        rc_log.insert(1, get_port(self.config_file))
+        rc_log = ''.join(rc_log)
+
+        server_out = open(rc_log, 'w')
+
+        command = ' '.join(self._args)
+        print('Starting rhodecode-vcsserver: {}'.format(host_url))
+        print('Command: {}'.format(command))
+        print('Logfile: {}'.format(rc_log))
+        self.process = subprocess32.Popen(
+            self._args, bufsize=0, env=env, stdout=server_out, stderr=server_out)
 
     def wait_until_ready(self, timeout=30):
         host = self._config['host']

rhodecode/tests/rhodecode.ini

@@ -46,27 +46,12 b' debug = true'
 host = 0.0.0.0
 port = 5000
 
-##################################
-##     WAITRESS WSGI SERVER     ##
-## Recommended for Development  ##
-##################################
-
-use = egg:waitress#main
-## number of worker threads
-threads = 5
-## MAX BODY SIZE 100GB
-max_request_body_size = 107374182400
-## Use poll instead of select, fixes file descriptors limits problems.
-## May not work on old windows systems.
-asyncore_use_poll = true
-
-
 ##########################
 ## GUNICORN WSGI SERVER ##
 ##########################
 ## run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
 
-#use = egg:gunicorn#main
+use = egg:gunicorn#main
 ## Sets the number of process workers. You must set `instance_id = *`
 ## when this option is set to more than one worker, recommended
 ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
@@ -90,66 +75,6 b' asyncore_use_poll = true'
 ## gets killed and restarted. Set to 6hrs
 #timeout = 21600
 
-## UWSGI ##
-## run with uwsgi --ini-paste-logged <inifile.ini>
-#[uwsgi]
-#socket = /tmp/uwsgi.sock
-#master = true
-#http = 127.0.0.1:5000
-
-## set as deamon and redirect all output to file
-#daemonize = ./uwsgi_rhodecode.log
-
-## master process PID
-#pidfile = ./uwsgi_rhodecode.pid
-
-## stats server with workers statistics, use uwsgitop
-## for monitoring, `uwsgitop 127.0.0.1:1717`
-#stats = 127.0.0.1:1717
-#memory-report = true
-
-## log 5XX errors
-#log-5xx = true
-
-## Set the socket listen queue size.
-#listen = 256
-
-## Gracefully Reload workers after the specified amount of managed requests
-## (avoid memory leaks).
-#max-requests = 1000
-
-## enable large buffers
-#buffer-size=65535
-
-## socket and http timeouts ##
-#http-timeout=3600
-#socket-timeout=3600
-
-## Log requests slower than the specified number of milliseconds.
-#log-slow = 10
-
-## Exit if no app can be loaded.
-#need-app = true
-
-## Set lazy mode (load apps in workers instead of master).
-#lazy = true
-
-## scaling ##
-## set cheaper algorithm to use, if not set default will be used
-#cheaper-algo = spare
-
-## minimum number of workers to keep at all times
-#cheaper = 1
-
-## number of workers to spawn at startup
-#cheaper-initial = 1
-
-## maximum number of workers that can be spawned
-#workers = 4
-
-## how many workers should be spawned at a time
-#cheaper-step = 1
-
 ## prefix middleware for RhodeCode.
 ## recommended when using proxy setup.
 ## allows to set RhodeCode under a prefix in server.

rhodecode/tests/vcs/conftest.py

@@ -18,8 +18,8 b''
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
+import time
 import shutil
-import time
 import datetime
 
 import pytest
@@ -29,7 +29,6 b' from rhodecode.lib.vcs.backends.base imp'
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.tests import get_new_dir
 from rhodecode.tests.utils import check_skip_backends, check_xfail_backends
-from rhodecode.tests.vcs.base import BackendTestMixin
 
 
 @pytest.fixture()
@@ -61,7 +60,7 b' def vcs_repository_support('
     cls.Backend = cls.backend_class = repo.__class__
     cls.imc = repo.in_memory_commit
 
-    return (backend_alias, repo)
+    return backend_alias, repo
 
 
 @pytest.fixture(scope='class')
@@ -88,6 +87,7 b' class VcsRepoContainer(object):'
     def get_repo(self, test_class, backend_alias):
         if backend_alias not in self._repos:
             repo = _create_empty_repository(test_class, backend_alias)
+
             self._cleanup_paths.append(repo.path)
             self._repos[backend_alias] = repo
         return self._repos[backend_alias]
@@ -106,7 +106,8 b' def _create_empty_repository(cls, backen'
     repo_path = get_new_dir(str(time.time()))
     repo = Backend(repo_path, create=True)
     if hasattr(cls, '_get_commits'):
-        cls.tip = _add_commits_to_repo(repo, cls._get_commits())
+        commits = cls._get_commits()
+        cls.tip = _add_commits_to_repo(repo, commits)
 
     return repo
 
@@ -132,7 +133,7 b' def config():'
 
 def _add_commits_to_repo(repo, commits):
     imc = repo.in_memory_commit
-    commit = None
+    tip = None
 
     for commit in commits:
         for node in commit.get('added', []):
@@ -142,13 +143,13 b' def _add_commits_to_repo(repo, commits):'
         for node in commit.get('removed', []):
             imc.remove(FileNode(node.path))
 
-        commit = imc.commit(
+        tip = imc.commit(
             message=unicode(commit['message']),
             author=unicode(commit['author']),
             date=commit['date'],
             branch=commit.get('branch'))
 
-    return commit
+    return tip
 
 
 @pytest.fixture
@@ -198,7 +199,7 b' def generate_repo_with_commits(vcs_repo)'
 def hg_repo(request, vcs_repo):
     repo = vcs_repo
 
-    commits = BackendTestMixin._get_commits()
+    commits = repo._get_commits()
     _add_commits_to_repo(repo, commits)
 
     return repo
@@ -207,3 +208,50 b' def hg_repo(request, vcs_repo):'
 @pytest.fixture
 def hg_commit(hg_repo):
     return hg_repo.get_commit()
+
+
+class BackendTestMixin(object):
+    """
+    This is a backend independent test case class which should be created
+    with ``type`` method.
+
+    It is required to set following attributes at subclass:
+
+    - ``backend_alias``: alias of used backend (see ``vcs.BACKENDS``)
+    - ``repo_path``: path to the repository which would be created for set of
+      tests
+    - ``recreate_repo_per_test``: If set to ``False``, repo would NOT be
+      created
+      before every single test. Defaults to ``True``.
+    """
+    recreate_repo_per_test = True
+
+    @classmethod
+    def _get_commits(cls):
+        commits = [
+            {
+                'message': u'Initial commit',
+                'author': u'Joe Doe <[e-mail]>',
+                'date': datetime.datetime(2010, 1, 1, 20),
+                'added': [
+                    FileNode('foobar', content='Foobar'),
+                    FileNode('foobar2', content='Foobar II'),
+                    FileNode('foo/bar/baz', content='baz here!'),
+                ],
+            },
+            {
+                'message': u'Changes...',
+                'author': u'Jane Doe <[e-mail]>',
+                'date': datetime.datetime(2010, 1, 1, 21),
+                'added': [
+                    FileNode('some/new.txt', content='news...'),
+                ],
+                'changed': [
+                    FileNode('foobar', 'Foobar I'),
+                ],
+                'removed': [],
+            },
+        ]
+        return commits
+
+

rhodecode/tests/vcs/test_archives.py

@@ -32,9 +32,10 b' import pytest'
 from rhodecode.lib.vcs.backends import base
 from rhodecode.lib.vcs.exceptions import ImproperArchiveTypeError, VCSError
 from rhodecode.lib.vcs.nodes import FileNode
-from rhodecode.tests.vcs.base import BackendTestMixin
+from rhodecode.tests.vcs.conftest import BackendTestMixin
 
 
[e-mail]("vcs_repository_support")
 class TestArchives(BackendTestMixin):
 
     @pytest.fixture(autouse=True)
@@ -48,7 +49,7 b' class TestArchives(BackendTestMixin):'
     @classmethod
     def _get_commits(cls):
         start_date = datetime.datetime(2010, 1, 1, 20)
-        for x in xrange(5):
+        for x in range(5):
             yield {
                 'message': 'Commit %d' % x,
                 'author': 'Joe Doe <[e-mail]>',
@@ -68,7 +69,7 b' class TestArchives(BackendTestMixin):'
         out_file.extractall(out_dir)
         out_file.close()
 
-        for x in xrange(5):
+        for x in range(5):
             node_path = '%d/file_%d.txt' % (x, x)
             with open(os.path.join(out_dir, 'repo/' + node_path)) as f:
                 file_content = f.read()
@@ -80,7 +81,7 b' class TestArchives(BackendTestMixin):'
         self.tip.archive_repo(self.temp_file, kind='zip', prefix='repo')
         out = zipfile.ZipFile(self.temp_file)
 
-        for x in xrange(5):
+        for x in range(5):
             node_path = '%d/file_%d.txt' % (x, x)
             decompressed = StringIO.StringIO()
             decompressed.write(out.read('repo/' + node_path))
@@ -98,7 +99,7 b' class TestArchives(BackendTestMixin):'
         raw_id = self.tip.raw_id
         assert 'rev:%s' % raw_id in metafile
 
-        for x in xrange(5):
+        for x in range(5):
             node_path = '%d/file_%d.txt' % (x, x)
             decompressed = StringIO.StringIO()
             decompressed.write(out.read('repo/' + node_path))

rhodecode/tests/vcs/test_branches.py

@@ -23,9 +23,10 b' import datetime'
 import pytest
 
 from rhodecode.lib.vcs.nodes import FileNode
-from rhodecode.tests.vcs.base import BackendTestMixin
+from rhodecode.tests.vcs.conftest import BackendTestMixin
 
 
[e-mail]("vcs_repository_support")
 class TestBranches(BackendTestMixin):
 
     def test_empty_repository_has_no_branches(self, vcsbackend):

rhodecode/tests/vcs/test_commits.py

@@ -32,7 +32,7 b' from rhodecode.lib.vcs.nodes import ('
     FileNode, AddedFileNodesGenerator,
     ChangedFileNodesGenerator, RemovedFileNodesGenerator)
 from rhodecode.tests import get_new_dir
-from rhodecode.tests.vcs.base import BackendTestMixin
+from rhodecode.tests.vcs.conftest import BackendTestMixin
 
 
 class TestBaseChangeset:
@@ -42,13 +42,14 b' class TestBaseChangeset:'
         pytest.deprecated_call(BaseChangeset)
 
 
-class TestEmptyCommit:
+class TestEmptyCommit(object):
 
     def test_branch_without_alias_returns_none(self):
         commit = EmptyCommit()
         assert commit.branch is None
 
 
[e-mail]("vcs_repository_support")
 class TestCommitsInNonEmptyRepo(BackendTestMixin):
     recreate_repo_per_test = True
 
@@ -197,6 +198,7 b' class TestCommitsInNonEmptyRepo(BackendT'
             assert [commit] == repo.get_commit(commit_idx=test_idx).children
 
 
[e-mail]("vcs_repository_support")
 class TestCommits(BackendTestMixin):
     recreate_repo_per_test = False
 
@@ -501,6 +503,7 b' def test_commit_is_link(vcsbackend, file'
     assert link_status is expected
 
 
[e-mail]("vcs_repository_support")
 class TestCommitsChanges(BackendTestMixin):
     recreate_repo_per_test = False
 

rhodecode/tests/vcs/test_diff.py

@@ -22,7 +22,7 b' import datetime'
 import pytest
 
 from rhodecode.lib.vcs.nodes import FileNode
-from rhodecode.tests.vcs.base import BackendTestMixin
+from rhodecode.tests.vcs.conftest import BackendTestMixin
 
 
 class TestGetDiffValidation:
@@ -68,6 +68,7 b' class TestGetDiffValidation:'
                 path='trunk/example.py', path1='branches/argparse/example.py')
 
 
[e-mail]("vcs_repository_support")
 class TestRepositoryGetDiff(BackendTestMixin):
 
     recreate_repo_per_test = False
@@ -405,6 +406,7 b' new file mode 10644'
         assert diff.raw == expected_diff
 
 
[e-mail]("vcs_repository_support")
 class TestGetDiffBinary(BackendTestMixin):
 
     recreate_repo_per_test = False

rhodecode/tests/vcs/test_filenodes_unicode_path.py

@@ -19,10 +19,13 b''
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import datetime
+
+import pytest
 from rhodecode.lib.vcs.nodes import FileNode
-from rhodecode.tests.vcs.base import BackendTestMixin
+from rhodecode.tests.vcs.conftest import BackendTestMixin
 
 
[e-mail]("vcs_repository_support")
 class TestFileNodeUnicodePath(BackendTestMixin):
 
     fname = 'ąśðąęłąć.txt'

rhodecode/tests/vcs/test_getitem.py

@@ -24,9 +24,10 b' import pytest'
 
 from rhodecode.lib.vcs.exceptions import CommitDoesNotExistError
 from rhodecode.lib.vcs.nodes import FileNode
-from rhodecode.tests.vcs.base import BackendTestMixin
+from rhodecode.tests.vcs.conftest import BackendTestMixin
 
 
[e-mail]("vcs_repository_support")
 class TestGetitem(BackendTestMixin):
 
     @classmethod

rhodecode/tests/vcs/test_getslice.py

@@ -19,10 +19,12 b''
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import datetime
 
+import pytest
 from rhodecode.lib.vcs.nodes import FileNode
-from rhodecode.tests.vcs.base import BackendTestMixin
+from rhodecode.tests.vcs.conftest import BackendTestMixin
 
 
[e-mail]("vcs_repository_support")
 class TestGetslice(BackendTestMixin):
 
     @classmethod

rhodecode/tests/vcs/test_git.py

@@ -35,7 +35,7 b' from rhodecode.lib.vcs.exceptions import'
 from rhodecode.lib.vcs.nodes import (
     NodeKind, FileNode, DirNode, NodeState, SubModuleNode)
 from rhodecode.tests import TEST_GIT_REPO, TEST_GIT_REPO_CLONE, get_new_dir
-from rhodecode.tests.vcs.base import BackendTestMixin
+from rhodecode.tests.vcs.conftest import BackendTestMixin
 
 
 pytestmark = pytest.mark.backends("git")
@@ -1030,6 +1030,7 b' class TestLargeFileRepo(object):'
         assert lf_node.name == '1MB.zip'
 
 
[e-mail]("vcs_repository_support")
 class TestGitSpecificWithRepo(BackendTestMixin):
 
     @classmethod
@@ -1092,6 +1093,7 b' class TestGitSpecificWithRepo(BackendTes'
              self.repo._get_commit_id(1), '--', 'foo'])
 
 
[e-mail]("vcs_repository_support")
 class TestGitRegression(BackendTestMixin):
 
     @classmethod

rhodecode/tests/vcs/test_inmemory.py

@@ -31,7 +31,7 b' from rhodecode.lib.vcs.exceptions import'
     NodeAlreadyRemovedError, NodeAlreadyChangedError, NodeDoesNotExistError,
     NodeNotChangedError)
 from rhodecode.lib.vcs.nodes import DirNode, FileNode
-from rhodecode.tests.vcs.base import BackendTestMixin
+from rhodecode.tests.vcs.conftest import BackendTestMixin
 
 
 @pytest.fixture
@@ -57,6 +57,7 b' def nodes():'
     return nodes
 
 
[e-mail]("vcs_repository_support")
 class TestInMemoryCommit(BackendTestMixin):
     """
     This is a backend independent test case class which should be created

rhodecode/tests/vcs/test_nodes.py

@@ -27,7 +27,7 b' from rhodecode.lib.vcs.nodes import File'
 from rhodecode.lib.vcs.nodes import Node
 from rhodecode.lib.vcs.nodes import NodeError
 from rhodecode.lib.vcs.nodes import NodeKind
-from rhodecode.tests.vcs.base import BackendTestMixin
+from rhodecode.tests.vcs.conftest import BackendTestMixin
 
 
 @pytest.fixture()
@@ -248,7 +248,7 b' class TestNodeBasics:'
         assert (1, 1) == py_node.lines(count_empty=True)
 
 
-class TestNodeContent:
+class TestNodeContent(object):
 
     def test_if_binary(self, binary_filenode):
         filenode = binary_filenode('calendar.jpg')
@@ -263,6 +263,7 b' class TestNodeContent:'
         assert tar_node.mimetype == 'application/x-tar'
 
 
[e-mail]("vcs_repository_support")
 class TestNodesCommits(BackendTestMixin):
 
     def test_node_last_commit(self, generate_repo_with_commits):

rhodecode/tests/vcs/test_repository.py

@@ -29,9 +29,10 b' from rhodecode.lib.vcs.backends.base imp'
     Config, BaseInMemoryCommit, Reference, MergeResponse, MergeFailureReason)
 from rhodecode.lib.vcs.exceptions import VCSError, RepositoryError
 from rhodecode.lib.vcs.nodes import FileNode
-from rhodecode.tests.vcs.base import BackendTestMixin
+from rhodecode.tests.vcs.conftest import BackendTestMixin
 
 
[e-mail]("vcs_repository_support")
 class TestRepositoryBase(BackendTestMixin):
     recreate_repo_per_test = False
 
@@ -148,6 +149,7 b' class TestRepositoryBase(BackendTestMixi'
         assert self.repo._get_url(url)
 
 
[e-mail]("vcs_repository_support")
 class TestDeprecatedRepositoryAPI(BackendTestMixin):
     recreate_repo_per_test = False
 
@@ -450,6 +452,7 b' class TestRepositoryMerge:'
                 ref, self, ref, 'workspace_id', 'user name', '[e-mail]')
 
 
[e-mail]("vcs_repository_support")
 class TestRepositoryStrip(BackendTestMixin):
     recreate_repo_per_test = True
 

rhodecode/tests/vcs/test_tags.py

@@ -20,7 +20,7 b''
 
 import pytest
 
-from rhodecode.tests.vcs.base import BackendTestMixin
+from rhodecode.tests.vcs.conftest import BackendTestMixin
 from rhodecode.lib.vcs.exceptions import (
     TagAlreadyExistError, TagDoesNotExistError)
 
@@ -28,6 +28,7 b' from rhodecode.lib.vcs.exceptions import'
 pytestmark = pytest.mark.backends("git", "hg")
 
 
[e-mail]("vcs_repository_support")
 class TestTags(BackendTestMixin):
 
     def test_new_tag(self):

rhodecode/tests/vcs/test_utils.py

@@ -18,8 +18,8 b''
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
+import os
 import datetime
-import os
 import subprocess32
 
 import pytest
@@ -33,7 +33,7 b' from rhodecode.lib.vcs.utils.paths impor'
 
 
 @pytest.mark.usefixtures("baseapp")
-class TestPaths:
+class TestPaths(object):
 
     def _test_get_dirs_for_path(self, path, expected):
         """
@@ -69,7 +69,7 b' class TestPaths:'
         assert set(get_scms_for_path(new)) == set(['git', 'hg'])
 
 
-class TestGetScm:
+class TestGetScm(object):
 
     def test_existing_repository(self, vcs_repository_support):
         alias, repo = vcs_repository_support
@@ -98,7 +98,7 b' class TestGetScm:'
             get_scm(tmpdir.strpath)
 
 
-class TestParseDatetime:
+class TestParseDatetime(object):
 
     def test_datetime_text(self):
         assert parse_datetime('2010-04-07 21:29:41') == \
@@ -186,7 +186,7 b' class TestParseDatetime:'
     ('Mr Double Name [e-mail] ',
      'Mr Double Name', '[e-mail]'),
 ])
-class TestAuthorExtractors:
+class TestAuthorExtractors(object):
 
     def test_author_email(self, test_str, name, email):
         assert email == author_email(test_str)

rhodecode/tests/vcsserver_http.ini

@@ -8,7 +8,6 b' use = egg:rhodecode-vcsserver'
 
 pyramid.default_locale_name = en
 pyramid.includes =
-pyramid.reload_templates = true
 
 # default locale used by VCS systems
 locale = en_US.UTF-8
@@ -22,10 +21,11 b' beaker.cache.repo_object.expire = 300'
 beaker.cache.repo_object.enabled = true
 
 [server:main]
-use = egg:waitress#main
 host = 127.0.0.1
 port = 9900
 
+use = egg:gunicorn#main
+
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################