retar correctly xz

setup.cfg

@@ -1,108 +1,108 b''
 [metadata]
 name = ipython
 version = attr: IPython.core.release.__version__
 url = https://ipython.org
 description = IPython: Productive Interactive Computing
 long_description_content_type = text/x-rst
 long_description = IPython provides a rich toolkit to help you make the most out of using Python
     interactively.  Its main components are:
 
     * A powerful interactive Python shell
     * A `Jupyter <https://jupyter.org/>`_ kernel to work with Python code in Jupyter
     notebooks and other interactive frontends.
 
     The enhanced interactive Python shells have the following main features:
 
     * Comprehensive object introspection.
 
     * Input history, persistent across sessions.
 
     * Caching of output results during a session with automatically generated
     references.
 
     * Extensible tab completion, with support by default for completion of python
     variables and keywords, filenames and function keywords.
 
     * Extensible system of 'magic' commands for controlling the environment and
     performing many tasks related either to IPython or the operating system.
 
     * A rich configuration system with easy switching between different setups
     (simpler than changing $PYTHONSTARTUP environment variables every time).
 
     * Session logging and reloading.
 
     * Extensible syntax processing for special purpose situations.
 
     * Access to the system shell with user-extensible alias system.
 
     * Easily embeddable in other Python programs and GUIs.
 
     * Integrated access to the pdb debugger and the Python profiler.
 
     The latest development version is always available from IPython's `GitHub
     site <http://github.com/ipython>`_.
 
 license_file = LICENSE
 project_urls =
     Documentation = https://ipython.readthedocs.io/
     Funding = https://numfocus.org/
     Source = https://github.com/ipython/ipython
     Tracker = https://github.com/ipython/ipython/issues
 keywords = Interactive, Interpreter, Shell, Embedding
 platforms = Linux, Mac OSX, Windows
 classifiers =
     Framework :: IPython
     Intended Audience :: Developers
     Intended Audience :: Science/Research
     License :: OSI Approved :: BSD License
     Programming Language :: Python
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3 :: Only
-    Topic :: System :: Shell
+    Topic :: System :: Shells
 
 
 [options]
 packages = find:
 python_requires = >=3.8
 zip_safe = False
 install_requires =
     setuptools>=18.5
     jedi>=0.16
     decorator
     pickleshare
     traitlets>=5
     prompt_toolkit>=2.0.0,<3.1.0,!=3.0.0,!=3.0.1
     pygments
     backcall
     stack_data
     matplotlib-inline
     pexpect>4.3; sys_platform != "win32"
     appnope; sys_platform == "darwin"
     colorama; sys_platform == "win32"
 
 [options.packages.find]
 exclude =
     setupext
 
 [options.package_data]
 IPython.core = profile/README*
 IPython.core.tests = *.png, *.jpg, daft_extension/*.py
 IPython.lib.tests = *.wav
 IPython.testing.plugin = *.txt
 
 [options.entry_points]
 console_scripts =
     ipython = IPython:start_ipython
     ipython3 = IPython:start_ipython
 pygments.lexers =
     ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer
     ipython = IPython.lib.lexers:IPythonLexer
     ipython3 = IPython.lib.lexers:IPython3Lexer
 
 [velin]
 ignore_patterns = 
    IPython/core/tests
    IPython/testing
 
 [tool.black]
 exclude = 'timing\.py'

tools/build_release

@@ -1,22 +1,23 b''
 #!/usr/bin/env python3
 """IPython release build script.
 """
 import os
 import sys
 from shutil import rmtree
 
 from toollib import sh, pjoin, get_ipdir, cd, sdists, buildwheels
 
 def build_release():
 
     # Get main ipython dir, this will raise if it doesn't pass some checks
     ipdir = get_ipdir()
     cd(ipdir)
 
     # Build source and binary distros
     sh(sdists)
     buildwheels()
     sh(' '.join([sys.executable, 'tools/retar.py', 'dist/*.gz']))
+    sh(' '.join([sys.executable, 'tools/retar.py', 'dist/*.xz']))
 
 if __name__ == '__main__':
     build_release()

tools/release

@@ -1,93 +1,93 b''
 #!/usr/bin/env python3
 """IPython release script.
 
 This should ONLY be run at real release time.
 """
 from __future__ import print_function
 
 import os
 from glob import glob
 from subprocess import call
 import sys
 
 from toollib import (get_ipdir, pjoin, cd, execfile, sh, archive,
         sdists, archive_user, archive_dir, buildwheels)
 from gh_api import post_download
 
 # Get main ipython dir, this will raise if it doesn't pass some checks
 ipdir = get_ipdir()
 tooldir = pjoin(ipdir, 'tools')
 distdir = pjoin(ipdir, 'dist')
 
 # Where I keep static backups of each release
 ipbackupdir = os.path.expanduser('~/ipython/backup')
 if not os.path.exists(ipbackupdir):
     os.makedirs(ipbackupdir)
 
 # Start in main IPython dir
 cd(ipdir)
 
 # Load release info
 version = None
 execfile(pjoin('IPython','core','release.py'), globals())
 
 # Build site addresses for file uploads
 release_site = '%s/release/%s' % (archive, version)
 backup_site =  '%s/backup/' % archive
 
 # Start actual release process
 print()
 print('Releasing IPython')
 print('=================')
 print()
 print('Version:', version)
 print()
 print('Source IPython directory:', ipdir)
 print()
 
 # Perform local backup, go to tools dir to run it.
 cd(tooldir)
 
 if 'upload' in sys.argv:
     cd(distdir)
 
     # do not upload OS specific files like .DS_Store
     to_upload = glob('*.whl')+glob('*.tar.gz')
     for fname in to_upload:
         # TODO: update to GitHub releases API
         continue
         print('uploading %s to GitHub' % fname)
         desc = "IPython %s source distribution" % version
         post_download("ipython/ipython", fname, description=desc)
 
     # Make target dir if it doesn't exist
     print('1. Uploading IPython to archive.ipython.org')
     sh('ssh %s "mkdir -p %s/release/%s" ' % (archive_user, archive_dir, version))
-    sh('scp *.tar.gz *.whl %s' % release_site)
+    sh('scp *.tar.gz *.tar.xz *.whl %s' % release_site)
 
     print('2. Uploading backup files...')
     cd(ipbackupdir)
     sh('scp `ls -1tr *tgz | tail -1` %s' % backup_site)
 
     print('3. Uploading to PyPI using twine')
     cd(distdir)
     call(['twine', 'upload'] + to_upload)
 
 else:
     # Build, but don't upload
 
     # Make backup tarball
     sh('./make_tarball.py')
     sh('mv ipython-*.tgz %s' % ipbackupdir)
 
     # Build release files
     sh('./build_release')
 
     cd(ipdir)
 
     buildwheels()
     print("`./release upload` to upload source distribution on PyPI and ipython archive")
     sys.exit(0)
 
 
 

tools/retar.py

@@ -1,68 +1,83 b''
 """
 Un-targz and retargz a targz file to ensure reproducible build. 
 
 usage: 
 
     $ export SOURCE_DATE_EPOCH=$(date +%s)
     ...
     $ python retar.py <tarfile.gz>
 
 The process of creating an sdist can be non-reproducible:
   - directory created during the process get a mtime of the creation date; 
   - gziping files embed the timestamp of zip creation. 
 
 This will untar-retar; ensuring that all mtime > SOURCE_DATE_EPOCH will be set
 equal to SOURCE_DATE_EPOCH.
 
 """
 
 import tarfile
 import sys
 import os
 import gzip
 import io
 
 from pathlib import Path
 
 if len(sys.argv) > 2:
     raise ValueError("Too many arguments")
 
 
 timestamp = int(os.environ["SOURCE_DATE_EPOCH"])
 
 path = Path(sys.argv[1])
 old_buf = io.BytesIO()
 with open(path, "rb") as f:
     old_buf.write(f.read())
 old_buf.seek(0)
-old = tarfile.open(fileobj=old_buf, mode="r:gz")
+if path.name.endswith("gz"):
+    r_mode = "r:gz"
+if path.name.endswith(("xz", "xz2")):
+    r_mode = "r:xz"
+old = tarfile.open(fileobj=old_buf, mode=r_mode)
 
 buf = io.BytesIO()
 new = tarfile.open(fileobj=buf, mode="w", format=tarfile.GNU_FORMAT)
 for i, m in enumerate(old):
     data = None
     # mutation does not work, copy
     if m.name.endswith('.DS_Store'):
         continue
     m2 = tarfile.TarInfo(m.name)
     m2.mtime = min(timestamp, m.mtime)
+    m2.pax_headers["mtime"] = m2.mtime
     m2.size = m.size
     m2.type = m.type
     m2.linkname = m.linkname
     m2.mode = m.mode
     if m.isdir():
         new.addfile(m2)
     else:
         data = old.extractfile(m)
         new.addfile(m2, data)
 new.close()
 old.close()
 
 buf.seek(0)
-with open(path, "wb") as f:
-    with gzip.GzipFile('', "wb", fileobj=f, mtime=timestamp) as gzf:
-        gzf.write(buf.read())
+
+if r_mode == "r:gz":
+    with open(path, "wb") as f:
+        with gzip.GzipFile("", "wb", fileobj=f, mtime=timestamp) as gzf:
+            gzf.write(buf.read())
+elif r_mode == "r:xz":
+    import lzma
+
+    with lzma.open(path, "wb") as f:
+        f.write(buf.read())
+
+else:
+    assert False
 
 # checks the archive is valid.
 archive = tarfile.open(path)
 names = archive.getnames()

tools/toollib.py

@@ -1,51 +1,51 b''
 """Various utilities common to IPython release and maintenance tools.
 """
 
 # Library imports
 import os
 import sys
 
 # Useful shorthands
 pjoin = os.path.join
 cd = os.chdir
 
 # Constants
 
 # SSH root address of the archive site
 archive_user = '[e-mail]'
 archive_dir = 'archive.ipython.org'
 archive = '%s:%s' % (archive_user, archive_dir)
 
 # Build commands
 # Source dists
-sdists = "{python} setup.py sdist --formats=xztar".format(python=sys.executable)
+sdists = "{python} setup.py sdist --formats=xztar,gztar".format(python=sys.executable)
 # Binary dists
 def buildwheels():
     sh("{python} setup.py bdist_wheel".format(python=sys.executable))
 
 
 # Utility functions
 def sh(cmd):
     """Run system command in shell, raise SystemExit if it returns an error."""
     print("$", cmd)
     stat = os.system(cmd)
     #stat = 0  # Uncomment this and comment previous to run in debug mode
     if stat:
         raise SystemExit("Command %s failed with code: %s" % (cmd, stat))
 
 def get_ipdir():
     """Get IPython directory from command line, or assume it's the one above."""
 
     # Initialize arguments and check location
     ipdir = pjoin(os.path.dirname(__file__), os.pardir)
 
     ipdir = os.path.abspath(ipdir)
 
     cd(ipdir)
     if not os.path.isdir('IPython') and os.path.isfile('setup.py'):
         raise SystemExit('Invalid ipython directory: %s' % ipdir)
     return ipdir
 
 def execfile(fname, globs, locs=None):
     locs = locs or globs
     exec(compile(open(fname).read(), fname, "exec"), globs, locs)