diff --git a/.travis.yml b/.travis.yml
index 47d42d6..faabb41 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,7 +12,7 @@ before_install:
     - git clone --quiet --depth 1 https://github.com/minrk/travis-wheels travis-wheels
     - 'if [[ $GROUP != js* ]]; then COVERAGE=""; fi'
 install:
-    - pip install "setuptools>=18.5"
+    - pip install "setuptools>=18.5" pip --upgrade
     # Installs PyPy (+ its Numpy). Based on @frol comment at:
     # https://github.com/travis-ci/travis-ci/issues/5027
     - |
diff --git a/setup.py b/setup.py
index 203840b..9cbb0ff 100755
--- a/setup.py
+++ b/setup.py
@@ -182,7 +182,7 @@ extras_require = dict(
     parallel = ['ipyparallel'],
     qtconsole = ['qtconsole'],
     doc = ['Sphinx>=1.3'],
-    test = ['nose>=0.10.1', 'requests', 'testpath', 'pygments', 'nbformat', 'ipykernel', 'numpy'],
+    test = ['nose>=0.10.1', 'requests', 'testpath', 'pygments', 'nbformat', 'ipykernel'],
     terminal = [],
     kernel = ['ipykernel'],
     nbformat = ['nbformat'],
@@ -207,6 +207,7 @@ install_requires = [
 extras_require.update({
     ':python_version == "2.7"': ['backports.shutil_get_terminal_size'],
     ':python_version == "2.7" or python_version == "3.3"': ['pathlib2'],
+    'test:python_version >= "3.4"': ['numpy'],
     ':sys_platform != "win32"': ['pexpect'],
     ':sys_platform == "darwin"': ['appnope'],
     ':sys_platform == "win32"': ['colorama'],