Update the release process to attempt reproducible builds.

This include some refactoring of the current scripts as well as the
introduction of a script to repack the tar.gz in order to remove a
couple of artifact the build environment. Mostly timestamps, extra
files, and groups ownership of files.

tools/retar.py

@@ -0,0 +1,60 b''
+"""
+Un-targz and retargz a targz file to ensure reproducible build. 
+
+usage: 
+
+    $ export SOURCE_DATE_EPOCH=$(date +%s)
+    ...
+    $ python retar.py <tarfile.gz>
+
+The process of creating an sdist can be non-reproducible:
+  - directory created during the process get a mtime of the creation date; 
+  - gziping files embed the timestamp of fo zip creation. 
+
+This will untar-retar; ensuring that all mtime > SOURCE_DATE_EPOCH will be set
+equal to SOURCE_DATE_EPOCH.
+
+"""
+
+import tarfile
+import sys
+import os
+import gzip
+import io
+
+if len(sys.argv) > 2:
+    raise ValueError("Too many arguments")
+
+
+timestamp = int(os.environ["SOURCE_DATE_EPOCH"])
+
+old_buf = io.BytesIO()
+with open(sys.argv[1], "rb") as f:
+    old_buf.write(f.read())
+old_buf.seek(0)
+old = tarfile.open(fileobj=old_buf, mode="r:gz")
+
+buf = io.BytesIO()
+new = tarfile.open(fileobj=buf, mode="w", format=tarfile.GNU_FORMAT)
+for i, m in enumerate(old):
+    data = None
+    # mutation does not work, copy
+    if m.name.endswith('.DS_Store'):
+        continue
+    m2 = tarfile.TarInfo(m.name)
+    m2.mtime = min(timestamp, m.mtime)
+    m2.size = m.size
+    m2.type = m.type
+    m2.linkname = m.linkname
+    if m.isdir():
+        data = old.extractfile(m)
+        new.addfile(m2, data)
+    else:
+        new.addfile(m2)
+new.close()
+old.close()
+
+buf.seek(0)
+with open(sys.argv[1], "wb") as f:
+    with gzip.GzipFile('', "wb", fileobj=f, mtime=timestamp) as gzf:
+        gzf.write(buf.read())

docs/source/coredev/index.rst

@@ -80,6 +80,13 b' for the release you are actually making::'
     VERSION=5.0.0
     BRANCH=master
 
+For `reproducibility of builds <https://reproducible-builds.org/specs/source-date-epoch/>`_,
+we recommend setting ``SOURCE_DATE_EPOCH`` prior to running the build; record the used value
+of ``SOURCE_DATE_EPOCH`` as it may not be available from build artifact. You
+should be able to use ``date +%s`` to get a formatted timestamp::
+
+    SOURCE_DATE_EPOCH=$(date +%s)
+
 
 2. Create GitHub stats and finish release note
 ----------------------------------------------
@@ -229,6 +236,16 b' uploading them to PyPI. We do not use an universal wheel as each wheel'
 installs an ``ipython2`` or ``ipython3`` script, depending on the version of
 Python it is built for. Using an universal wheel would prevent this.
 
+Check the shasum of files with::
+
+    shasum -a 256 dist/*
+
+and takes notes of them you might need them to update the conda-forge recipes.
+Rerun the command and check the hash have not changed::
+
+    ./tools/release
+    shasum -a 256 dist/*
+
 Use the following to actually upload the result of the build::
 
     ./tools/release upload

setup.py

@@ -224,7 +224,7 b' everything = set()'
 for key, deps in extras_require.items():
     if ':' not in key:
         everything.update(deps)
-extras_require['all'] = everything
+extras_require['all'] = list(sorted(everything))
 
 if 'setuptools' in sys.modules:
     setuptools_extra_args['python_requires'] = '>=3.6'

tools/build_release

@@ -2,6 +2,7 b''
 """IPython release build script.
 """
 import os
+import sys
 from shutil import rmtree
 
 from toollib import sh, pjoin, get_ipdir, cd, sdists, buildwheels
@@ -12,15 +13,10 b' def build_release():'
     ipdir = get_ipdir()
     cd(ipdir)
 
-    # Cleanup
-    for d in ['build', 'dist', pjoin('docs', 'build'), pjoin('docs', 'dist'),
-              pjoin('docs', 'source', 'api', 'generated')]:
-        if os.path.isdir(d):
-            rmtree(d)
-
     # Build source and binary distros
     sh(sdists)
     buildwheels()
+    sh(' '.join([sys.executable, 'tools/retar.py', 'dist/*.gz']))
 
 if __name__ == '__main__':
     build_release()

tools/make_tarball.py

@@ -3,7 +3,6 b''
 """
 
 import subprocess
-import os
 
 from toollib import cd, sh
 

tools/release

@@ -81,13 +81,10 b' else:'
     sh('mv ipython-*.tgz %s' % ipbackupdir)
 
     # Build release files
-    sh('./build_release %s' % ipdir)
+    sh('./build_release')
 
     cd(ipdir)
 
-    # Upload all files
-    sh(sdists)
-
     buildwheels()
     print("`./release upload` to upload source distribution on PyPI and ipython archive")
     sys.exit(0)

tools/release_helper.sh

@@ -98,12 +98,6 b' then'
 
 fi
 
-echo
-echo $BLUE"Attempting to build package..."$NOR
-
-tools/build_release
-rm dist/*
-
 if ask_section "Should we commit, tag, push... etc ? "
 then
    echo
@@ -161,13 +155,40 b' fi'
 if ask_section "Should we build and release ?"
 then
 
+    echo $BLUE"going to set SOURCE_DATE_EPOCH"$NOR
+    echo $BLUE'export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)'$NOR
+    echo $GREEN"Press enter to continue"$NOR
+    read
+
+    export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)
+
+    echo $BLUE"SOURCE_DATE_EPOCH set to $SOURCE_DATE_EPOCH"$NOR
+    echo $GREEN"Press enter to continue"$NOR
+    read
+
+
+
+    echo
+    echo $BLUE"Attempting to build package..."$NOR
+
+    tools/release
+
+
+    echo $RED'$ shasum -a 256 dist/*'
+    shasum -a 256 dist/*
+    echo $NOR
+
+    echo $BLUE"We are going to rebuild, node the hash above, and compare them to the rebuild"$NOR
+    echo $GREEN"Press enter to continue"$NOR
+    read
+
     echo
     echo $BLUE"Attempting to build package..."$NOR
 
     tools/release
 
-    echo $RED
-    echo '$ shasum -a 256 dist/*'
+    echo $RED"Check the shasum for SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
+    echo $RED'$ shasum -a 256 dist/*'
     shasum -a 256 dist/*
     echo $NOR