diff --git a/docs/source/coredev/index.rst b/docs/source/coredev/index.rst
index 60f1cb0..ee1eadb 100644
--- a/docs/source/coredev/index.rst
+++ b/docs/source/coredev/index.rst
@@ -80,6 +80,13 @@ for the release you are actually making::
VERSION=5.0.0
BRANCH=master
+For `reproducibility of builds `_,
+we recommend setting ``SOURCE_DATE_EPOCH`` prior to running the build; record the used value
+of ``SOURCE_DATE_EPOCH`` as it may not be available from build artifact. You
+should be able to use ``date +%s`` to get a formatted timestamp::
+
+ SOURCE_DATE_EPOCH=$(date +%s)
+
2. Create GitHub stats and finish release note
----------------------------------------------
@@ -229,6 +236,16 @@ uploading them to PyPI. We do not use an universal wheel as each wheel
installs an ``ipython2`` or ``ipython3`` script, depending on the version of
Python it is built for. Using an universal wheel would prevent this.
+Check the shasum of files with::
+
+ shasum -a 256 dist/*
+
+and takes notes of them you might need them to update the conda-forge recipes.
+Rerun the command and check the hash have not changed::
+
+ ./tools/release
+ shasum -a 256 dist/*
+
Use the following to actually upload the result of the build::
./tools/release upload
diff --git a/setup.py b/setup.py
index 9ec67cb..29d7260 100755
--- a/setup.py
+++ b/setup.py
@@ -224,7 +224,7 @@ everything = set()
for key, deps in extras_require.items():
if ':' not in key:
everything.update(deps)
-extras_require['all'] = everything
+extras_require['all'] = list(sorted(everything))
if 'setuptools' in sys.modules:
setuptools_extra_args['python_requires'] = '>=3.6'
diff --git a/tools/build_release b/tools/build_release
index 26dc9ec..51fd87d 100755
--- a/tools/build_release
+++ b/tools/build_release
@@ -2,6 +2,7 @@
"""IPython release build script.
"""
import os
+import sys
from shutil import rmtree
from toollib import sh, pjoin, get_ipdir, cd, sdists, buildwheels
@@ -12,15 +13,10 @@ def build_release():
ipdir = get_ipdir()
cd(ipdir)
- # Cleanup
- for d in ['build', 'dist', pjoin('docs', 'build'), pjoin('docs', 'dist'),
- pjoin('docs', 'source', 'api', 'generated')]:
- if os.path.isdir(d):
- rmtree(d)
-
# Build source and binary distros
sh(sdists)
buildwheels()
+ sh(' '.join([sys.executable, 'tools/retar.py', 'dist/*.gz']))
if __name__ == '__main__':
build_release()
diff --git a/tools/make_tarball.py b/tools/make_tarball.py
index bdce25b..fb639f6 100755
--- a/tools/make_tarball.py
+++ b/tools/make_tarball.py
@@ -3,7 +3,6 @@
"""
import subprocess
-import os
from toollib import cd, sh
diff --git a/tools/release b/tools/release
index 5c8686b..2de8e12 100755
--- a/tools/release
+++ b/tools/release
@@ -81,13 +81,10 @@ else:
sh('mv ipython-*.tgz %s' % ipbackupdir)
# Build release files
- sh('./build_release %s' % ipdir)
+ sh('./build_release')
cd(ipdir)
- # Upload all files
- sh(sdists)
-
buildwheels()
print("`./release upload` to upload source distribution on PyPI and ipython archive")
sys.exit(0)
diff --git a/tools/release_helper.sh b/tools/release_helper.sh
index 7489c16..6e2e8bf 100644
--- a/tools/release_helper.sh
+++ b/tools/release_helper.sh
@@ -98,12 +98,6 @@ then
fi
-echo
-echo $BLUE"Attempting to build package..."$NOR
-
-tools/build_release
-rm dist/*
-
if ask_section "Should we commit, tag, push... etc ? "
then
echo
@@ -160,14 +154,41 @@ fi
if ask_section "Should we build and release ?"
then
+
+ echo $BLUE"going to set SOURCE_DATE_EPOCH"$NOR
+ echo $BLUE'export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)'$NOR
+ echo $GREEN"Press enter to continue"$NOR
+ read
+
+ export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)
+
+ echo $BLUE"SOURCE_DATE_EPOCH set to $SOURCE_DATE_EPOCH"$NOR
+ echo $GREEN"Press enter to continue"$NOR
+ read
+
+
+
+ echo
+ echo $BLUE"Attempting to build package..."$NOR
+
+ tools/release
+
+
+ echo $RED'$ shasum -a 256 dist/*'
+ shasum -a 256 dist/*
+ echo $NOR
+
+ echo $BLUE"We are going to rebuild, node the hash above, and compare them to the rebuild"$NOR
+ echo $GREEN"Press enter to continue"$NOR
+ read
echo
echo $BLUE"Attempting to build package..."$NOR
tools/release
- echo $RED
- echo '$ shasum -a 256 dist/*'
+ echo $RED"Check the shasum for SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
+ echo $RED'$ shasum -a 256 dist/*'
shasum -a 256 dist/*
echo $NOR
diff --git a/tools/retar.py b/tools/retar.py
new file mode 100644
index 0000000..dbdca0e
--- /dev/null
+++ b/tools/retar.py
@@ -0,0 +1,60 @@
+"""
+Un-targz and retargz a targz file to ensure reproducible build.
+
+usage:
+
+ $ export SOURCE_DATE_EPOCH=$(date +%s)
+ ...
+ $ python retar.py
+
+The process of creating an sdist can be non-reproducible:
+ - directory created during the process get a mtime of the creation date;
+ - gziping files embed the timestamp of fo zip creation.
+
+This will untar-retar; ensuring that all mtime > SOURCE_DATE_EPOCH will be set
+equal to SOURCE_DATE_EPOCH.
+
+"""
+
+import tarfile
+import sys
+import os
+import gzip
+import io
+
+if len(sys.argv) > 2:
+ raise ValueError("Too many arguments")
+
+
+timestamp = int(os.environ["SOURCE_DATE_EPOCH"])
+
+old_buf = io.BytesIO()
+with open(sys.argv[1], "rb") as f:
+ old_buf.write(f.read())
+old_buf.seek(0)
+old = tarfile.open(fileobj=old_buf, mode="r:gz")
+
+buf = io.BytesIO()
+new = tarfile.open(fileobj=buf, mode="w", format=tarfile.GNU_FORMAT)
+for i, m in enumerate(old):
+ data = None
+ # mutation does not work, copy
+ if m.name.endswith('.DS_Store'):
+ continue
+ m2 = tarfile.TarInfo(m.name)
+ m2.mtime = min(timestamp, m.mtime)
+ m2.size = m.size
+ m2.type = m.type
+ m2.linkname = m.linkname
+ if m.isdir():
+ data = old.extractfile(m)
+ new.addfile(m2, data)
+ else:
+ new.addfile(m2)
+new.close()
+old.close()
+
+buf.seek(0)
+with open(sys.argv[1], "wb") as f:
+ with gzip.GzipFile('', "wb", fileobj=f, mtime=timestamp) as gzf:
+ gzf.write(buf.read())