diff --git a/Dockerfile b/Dockerfile
index 9c34be4..7abe6ae 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -6,7 +6,10 @@ FROM ipython/scipystack
 
 MAINTAINER IPython Project <ipython-dev@scipy.org>
 
-ADD . /srv/ipython/
+# Can't directly add the source as it won't have the submodules
+RUN mkdir /srv/
+WORKDIR /srv/
+RUN git clone --recursive https://github.com/ipython/ipython.git
 WORKDIR /srv/ipython/
 
 # Installing certain dependencies directly