From c28577dd8446475856e8e598a9ccac2bb4d30531 2020-05-29 20:21:57 From: Matthias Bussonnier Date: 2020-05-29 20:21:57 Subject: [PATCH] Merge pull request #12351 from meeseeksmachine/auto-backport-of-pr-12347-on-7.x Backport PR #12347 on branch 7.x (What's new in 7.15+ Reproducible build working) --- diff --git a/docs/source/coredev/index.rst b/docs/source/coredev/index.rst index 60f1cb0..ee1eadb 100644 --- a/docs/source/coredev/index.rst +++ b/docs/source/coredev/index.rst @@ -80,6 +80,13 @@ for the release you are actually making:: VERSION=5.0.0 BRANCH=master +For `reproducibility of builds `_, +we recommend setting ``SOURCE_DATE_EPOCH`` prior to running the build; record the used value +of ``SOURCE_DATE_EPOCH`` as it may not be available from build artifact. You +should be able to use ``date +%s`` to get a formatted timestamp:: + + SOURCE_DATE_EPOCH=$(date +%s) + 2. Create GitHub stats and finish release note ---------------------------------------------- @@ -229,6 +236,16 @@ uploading them to PyPI. We do not use an universal wheel as each wheel installs an ``ipython2`` or ``ipython3`` script, depending on the version of Python it is built for. Using an universal wheel would prevent this. +Check the shasum of files with:: + + shasum -a 256 dist/* + +and takes notes of them you might need them to update the conda-forge recipes. +Rerun the command and check the hash have not changed:: + + ./tools/release + shasum -a 256 dist/* + Use the following to actually upload the result of the build:: ./tools/release upload diff --git a/docs/source/whatsnew/version7.rst b/docs/source/whatsnew/version7.rst index f68c810..c35a465 100644 --- a/docs/source/whatsnew/version7.rst +++ b/docs/source/whatsnew/version7.rst @@ -2,6 +2,90 @@ 7.x Series ============ +.. _version 715: + +IPython 7.15 +============ + +IPython 7.15 brings a number of bug fixes and user facing improvements. + +Misc Noticeable changes: +------------------------ + + - Long completion name have better elision in terminal :ghpull:`12284` + - I've started to test on Python 3.9 :ghpull:`12307` and fix some errors. + - Hi DPI scaling of figures when using qt eventloop :ghpull:`12314` + - Document the ability to have systemwide configuration for IPython. + :ghpull:`12328` + - Fix issues with input autoformatting :ghpull:`12336` + +Reproducible Build +------------------ + +Starting with IPython 7.15, I am attempting to provide reproducible builds, +that is to say you should be able from the source tree to generate an sdist +and wheel that are identical byte for byte with the publish version on PyPI. + +I've only tested on a couple of machines so far and the process is relatively +straightforward, so this mean that IPython not only have a deterministic build +process, but also I have either removed, or put under control all effects of +the build environments on the final artifact. I encourage you to attempt the +build process on your machine as documented in :ref:`core_developer_guide` +and let me know if you do not obtain an identical artifact. + +While reproducible builds is critical to check that the supply chain of (open +source) software has not been compromised, it can also help to speedup many +of the build processes in large environment (conda, apt...) by allowing +better caching of intermediate build steps. + +Learn more on ``_. `Reflections on trusting +trust `_ is also one of the +cornerstone and recommended reads on this subject. + +.. note:: + + The build commit from which the sdist is generated is also `signed + `_, so you should be able to + check it has not been compromised, and the git repository is a `merkle-tree + `_, you can check the consistency + with `git-fsck `_ which you likely `want + to enable by default + `_. + +NEP29: Last version to support Python 3.6 +----------------------------------------- + +IPython 7.15 will be the Last IPython version to officially support Python +3.6, as stated by `NumPy Enhancement Proposal 29 +`_. Starting with +next minor version of IPython I may stop testing on Python 3.6 and may stop +publishing release artifacts that install on Python 3.6 + +Highlighted features +-------------------- + +Highlighted features are not new, but seem to not be widely known, this +section will help you discover in more narrative form what you can do with +IPython. + +Increase Tab Completion Menu Height +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In terminal IPython it is possible to increase the hight of the tab-completion +menu. To do so set the value of +:configtrait:`TerminalInteractiveShell.space_for_menu`, this will reserve more +space at the bottom of the screen for various kind of menus in IPython including +tab completion and searching in history. + +Autoformat Code in the terminal +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you have a preferred code formatter, you can configure IPython to +reformat your code. Set the value of +:configtrait:`TerminalInteractiveShell.autoformatter` to for example ``'black'`` +and IPython will auto format your code when possible. + + .. _version 714: IPython 7.14 diff --git a/setup.py b/setup.py index 593e3a6..daf5f55 100755 --- a/setup.py +++ b/setup.py @@ -225,7 +225,7 @@ everything = set() for key, deps in extras_require.items(): if ':' not in key: everything.update(deps) -extras_require['all'] = everything +extras_require['all'] = list(sorted(everything)) if 'setuptools' in sys.modules: setuptools_extra_args['python_requires'] = '>=3.6' diff --git a/tools/build_release b/tools/build_release index 26dc9ec..51fd87d 100755 --- a/tools/build_release +++ b/tools/build_release @@ -2,6 +2,7 @@ """IPython release build script. """ import os +import sys from shutil import rmtree from toollib import sh, pjoin, get_ipdir, cd, sdists, buildwheels @@ -12,15 +13,10 @@ def build_release(): ipdir = get_ipdir() cd(ipdir) - # Cleanup - for d in ['build', 'dist', pjoin('docs', 'build'), pjoin('docs', 'dist'), - pjoin('docs', 'source', 'api', 'generated')]: - if os.path.isdir(d): - rmtree(d) - # Build source and binary distros sh(sdists) buildwheels() + sh(' '.join([sys.executable, 'tools/retar.py', 'dist/*.gz'])) if __name__ == '__main__': build_release() diff --git a/tools/make_tarball.py b/tools/make_tarball.py index bdce25b..fb639f6 100755 --- a/tools/make_tarball.py +++ b/tools/make_tarball.py @@ -3,7 +3,6 @@ """ import subprocess -import os from toollib import cd, sh diff --git a/tools/release b/tools/release index 5c8686b..2de8e12 100755 --- a/tools/release +++ b/tools/release @@ -81,13 +81,10 @@ else: sh('mv ipython-*.tgz %s' % ipbackupdir) # Build release files - sh('./build_release %s' % ipdir) + sh('./build_release') cd(ipdir) - # Upload all files - sh(sdists) - buildwheels() print("`./release upload` to upload source distribution on PyPI and ipython archive") sys.exit(0) diff --git a/tools/release_helper.sh b/tools/release_helper.sh index 7489c16..6e2e8bf 100644 --- a/tools/release_helper.sh +++ b/tools/release_helper.sh @@ -98,12 +98,6 @@ then fi -echo -echo $BLUE"Attempting to build package..."$NOR - -tools/build_release -rm dist/* - if ask_section "Should we commit, tag, push... etc ? " then echo @@ -160,14 +154,41 @@ fi if ask_section "Should we build and release ?" then + + echo $BLUE"going to set SOURCE_DATE_EPOCH"$NOR + echo $BLUE'export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)'$NOR + echo $GREEN"Press enter to continue"$NOR + read + + export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD) + + echo $BLUE"SOURCE_DATE_EPOCH set to $SOURCE_DATE_EPOCH"$NOR + echo $GREEN"Press enter to continue"$NOR + read + + + + echo + echo $BLUE"Attempting to build package..."$NOR + + tools/release + + + echo $RED'$ shasum -a 256 dist/*' + shasum -a 256 dist/* + echo $NOR + + echo $BLUE"We are going to rebuild, node the hash above, and compare them to the rebuild"$NOR + echo $GREEN"Press enter to continue"$NOR + read echo echo $BLUE"Attempting to build package..."$NOR tools/release - echo $RED - echo '$ shasum -a 256 dist/*' + echo $RED"Check the shasum for SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" + echo $RED'$ shasum -a 256 dist/*' shasum -a 256 dist/* echo $NOR diff --git a/tools/retar.py b/tools/retar.py new file mode 100644 index 0000000..dbdca0e --- /dev/null +++ b/tools/retar.py @@ -0,0 +1,60 @@ +""" +Un-targz and retargz a targz file to ensure reproducible build. + +usage: + + $ export SOURCE_DATE_EPOCH=$(date +%s) + ... + $ python retar.py + +The process of creating an sdist can be non-reproducible: + - directory created during the process get a mtime of the creation date; + - gziping files embed the timestamp of fo zip creation. + +This will untar-retar; ensuring that all mtime > SOURCE_DATE_EPOCH will be set +equal to SOURCE_DATE_EPOCH. + +""" + +import tarfile +import sys +import os +import gzip +import io + +if len(sys.argv) > 2: + raise ValueError("Too many arguments") + + +timestamp = int(os.environ["SOURCE_DATE_EPOCH"]) + +old_buf = io.BytesIO() +with open(sys.argv[1], "rb") as f: + old_buf.write(f.read()) +old_buf.seek(0) +old = tarfile.open(fileobj=old_buf, mode="r:gz") + +buf = io.BytesIO() +new = tarfile.open(fileobj=buf, mode="w", format=tarfile.GNU_FORMAT) +for i, m in enumerate(old): + data = None + # mutation does not work, copy + if m.name.endswith('.DS_Store'): + continue + m2 = tarfile.TarInfo(m.name) + m2.mtime = min(timestamp, m.mtime) + m2.size = m.size + m2.type = m.type + m2.linkname = m.linkname + if m.isdir(): + data = old.extractfile(m) + new.addfile(m2, data) + else: + new.addfile(m2) +new.close() +old.close() + +buf.seek(0) +with open(sys.argv[1], "wb") as f: + with gzip.GzipFile('', "wb", fileobj=f, mtime=timestamp) as gzf: + gzf.write(buf.read())