Show More
@@ -0,0 +1,60 b'' | |||||
|
1 | """ | |||
|
2 | Un-targz and retargz a targz file to ensure reproducible build. | |||
|
3 | ||||
|
4 | usage: | |||
|
5 | ||||
|
6 | $ export SOURCE_DATE_EPOCH=$(date +%s) | |||
|
7 | ... | |||
|
8 | $ python retar.py <tarfile.gz> | |||
|
9 | ||||
|
10 | The process of creating an sdist can be non-reproducible: | |||
|
11 | - directory created during the process get a mtime of the creation date; | |||
|
12 | - gziping files embed the timestamp of fo zip creation. | |||
|
13 | ||||
|
14 | This will untar-retar; ensuring that all mtime > SOURCE_DATE_EPOCH will be set | |||
|
15 | equal to SOURCE_DATE_EPOCH. | |||
|
16 | ||||
|
17 | """ | |||
|
18 | ||||
|
19 | import tarfile | |||
|
20 | import sys | |||
|
21 | import os | |||
|
22 | import gzip | |||
|
23 | import io | |||
|
24 | ||||
|
25 | if len(sys.argv) > 2: | |||
|
26 | raise ValueError("Too many arguments") | |||
|
27 | ||||
|
28 | ||||
|
29 | timestamp = int(os.environ["SOURCE_DATE_EPOCH"]) | |||
|
30 | ||||
|
31 | old_buf = io.BytesIO() | |||
|
32 | with open(sys.argv[1], "rb") as f: | |||
|
33 | old_buf.write(f.read()) | |||
|
34 | old_buf.seek(0) | |||
|
35 | old = tarfile.open(fileobj=old_buf, mode="r:gz") | |||
|
36 | ||||
|
37 | buf = io.BytesIO() | |||
|
38 | new = tarfile.open(fileobj=buf, mode="w", format=tarfile.GNU_FORMAT) | |||
|
39 | for i, m in enumerate(old): | |||
|
40 | data = None | |||
|
41 | # mutation does not work, copy | |||
|
42 | if m.name.endswith('.DS_Store'): | |||
|
43 | continue | |||
|
44 | m2 = tarfile.TarInfo(m.name) | |||
|
45 | m2.mtime = min(timestamp, m.mtime) | |||
|
46 | m2.size = m.size | |||
|
47 | m2.type = m.type | |||
|
48 | m2.linkname = m.linkname | |||
|
49 | if m.isdir(): | |||
|
50 | data = old.extractfile(m) | |||
|
51 | new.addfile(m2, data) | |||
|
52 | else: | |||
|
53 | new.addfile(m2) | |||
|
54 | new.close() | |||
|
55 | old.close() | |||
|
56 | ||||
|
57 | buf.seek(0) | |||
|
58 | with open(sys.argv[1], "wb") as f: | |||
|
59 | with gzip.GzipFile('', "wb", fileobj=f, mtime=timestamp) as gzf: | |||
|
60 | gzf.write(buf.read()) |
@@ -80,6 +80,13 b' for the release you are actually making::' | |||||
80 | VERSION=5.0.0 |
|
80 | VERSION=5.0.0 | |
81 | BRANCH=master |
|
81 | BRANCH=master | |
82 |
|
82 | |||
|
83 | For `reproducibility of builds <https://reproducible-builds.org/specs/source-date-epoch/>`_, | |||
|
84 | we recommend setting ``SOURCE_DATE_EPOCH`` prior to running the build; record the used value | |||
|
85 | of ``SOURCE_DATE_EPOCH`` as it may not be available from build artifact. You | |||
|
86 | should be able to use ``date +%s`` to get a formatted timestamp:: | |||
|
87 | ||||
|
88 | SOURCE_DATE_EPOCH=$(date +%s) | |||
|
89 | ||||
83 |
|
90 | |||
84 | 2. Create GitHub stats and finish release note |
|
91 | 2. Create GitHub stats and finish release note | |
85 | ---------------------------------------------- |
|
92 | ---------------------------------------------- | |
@@ -229,6 +236,16 b' uploading them to PyPI. We do not use an universal wheel as each wheel' | |||||
229 | installs an ``ipython2`` or ``ipython3`` script, depending on the version of |
|
236 | installs an ``ipython2`` or ``ipython3`` script, depending on the version of | |
230 | Python it is built for. Using an universal wheel would prevent this. |
|
237 | Python it is built for. Using an universal wheel would prevent this. | |
231 |
|
238 | |||
|
239 | Check the shasum of files with:: | |||
|
240 | ||||
|
241 | shasum -a 256 dist/* | |||
|
242 | ||||
|
243 | and takes notes of them you might need them to update the conda-forge recipes. | |||
|
244 | Rerun the command and check the hash have not changed:: | |||
|
245 | ||||
|
246 | ./tools/release | |||
|
247 | shasum -a 256 dist/* | |||
|
248 | ||||
232 | Use the following to actually upload the result of the build:: |
|
249 | Use the following to actually upload the result of the build:: | |
233 |
|
250 | |||
234 | ./tools/release upload |
|
251 | ./tools/release upload |
@@ -224,7 +224,7 b' everything = set()' | |||||
224 | for key, deps in extras_require.items(): |
|
224 | for key, deps in extras_require.items(): | |
225 | if ':' not in key: |
|
225 | if ':' not in key: | |
226 | everything.update(deps) |
|
226 | everything.update(deps) | |
227 | extras_require['all'] = everything |
|
227 | extras_require['all'] = list(sorted(everything)) | |
228 |
|
228 | |||
229 | if 'setuptools' in sys.modules: |
|
229 | if 'setuptools' in sys.modules: | |
230 | setuptools_extra_args['python_requires'] = '>=3.6' |
|
230 | setuptools_extra_args['python_requires'] = '>=3.6' |
@@ -2,6 +2,7 b'' | |||||
2 | """IPython release build script. |
|
2 | """IPython release build script. | |
3 | """ |
|
3 | """ | |
4 | import os |
|
4 | import os | |
|
5 | import sys | |||
5 | from shutil import rmtree |
|
6 | from shutil import rmtree | |
6 |
|
7 | |||
7 | from toollib import sh, pjoin, get_ipdir, cd, sdists, buildwheels |
|
8 | from toollib import sh, pjoin, get_ipdir, cd, sdists, buildwheels | |
@@ -12,15 +13,10 b' def build_release():' | |||||
12 | ipdir = get_ipdir() |
|
13 | ipdir = get_ipdir() | |
13 | cd(ipdir) |
|
14 | cd(ipdir) | |
14 |
|
15 | |||
15 | # Cleanup |
|
|||
16 | for d in ['build', 'dist', pjoin('docs', 'build'), pjoin('docs', 'dist'), |
|
|||
17 | pjoin('docs', 'source', 'api', 'generated')]: |
|
|||
18 | if os.path.isdir(d): |
|
|||
19 | rmtree(d) |
|
|||
20 |
|
||||
21 | # Build source and binary distros |
|
16 | # Build source and binary distros | |
22 | sh(sdists) |
|
17 | sh(sdists) | |
23 | buildwheels() |
|
18 | buildwheels() | |
|
19 | sh(' '.join([sys.executable, 'tools/retar.py', 'dist/*.gz'])) | |||
24 |
|
20 | |||
25 | if __name__ == '__main__': |
|
21 | if __name__ == '__main__': | |
26 | build_release() |
|
22 | build_release() |
@@ -3,7 +3,6 b'' | |||||
3 | """ |
|
3 | """ | |
4 |
|
4 | |||
5 | import subprocess |
|
5 | import subprocess | |
6 | import os |
|
|||
7 |
|
6 | |||
8 | from toollib import cd, sh |
|
7 | from toollib import cd, sh | |
9 |
|
8 |
@@ -81,13 +81,10 b' else:' | |||||
81 | sh('mv ipython-*.tgz %s' % ipbackupdir) |
|
81 | sh('mv ipython-*.tgz %s' % ipbackupdir) | |
82 |
|
82 | |||
83 | # Build release files |
|
83 | # Build release files | |
84 |
sh('./build_release |
|
84 | sh('./build_release') | |
85 |
|
85 | |||
86 | cd(ipdir) |
|
86 | cd(ipdir) | |
87 |
|
87 | |||
88 | # Upload all files |
|
|||
89 | sh(sdists) |
|
|||
90 |
|
||||
91 | buildwheels() |
|
88 | buildwheels() | |
92 | print("`./release upload` to upload source distribution on PyPI and ipython archive") |
|
89 | print("`./release upload` to upload source distribution on PyPI and ipython archive") | |
93 | sys.exit(0) |
|
90 | sys.exit(0) |
@@ -98,12 +98,6 b' then' | |||||
98 |
|
98 | |||
99 | fi |
|
99 | fi | |
100 |
|
100 | |||
101 | echo |
|
|||
102 | echo $BLUE"Attempting to build package..."$NOR |
|
|||
103 |
|
||||
104 | tools/build_release |
|
|||
105 | rm dist/* |
|
|||
106 |
|
||||
107 | if ask_section "Should we commit, tag, push... etc ? " |
|
101 | if ask_section "Should we commit, tag, push... etc ? " | |
108 | then |
|
102 | then | |
109 | echo |
|
103 | echo | |
@@ -161,13 +155,40 b' fi' | |||||
161 | if ask_section "Should we build and release ?" |
|
155 | if ask_section "Should we build and release ?" | |
162 | then |
|
156 | then | |
163 |
|
157 | |||
|
158 | echo $BLUE"going to set SOURCE_DATE_EPOCH"$NOR | |||
|
159 | echo $BLUE'export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)'$NOR | |||
|
160 | echo $GREEN"Press enter to continue"$NOR | |||
|
161 | read | |||
|
162 | ||||
|
163 | export SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD) | |||
|
164 | ||||
|
165 | echo $BLUE"SOURCE_DATE_EPOCH set to $SOURCE_DATE_EPOCH"$NOR | |||
|
166 | echo $GREEN"Press enter to continue"$NOR | |||
|
167 | read | |||
|
168 | ||||
|
169 | ||||
|
170 | ||||
|
171 | echo | |||
|
172 | echo $BLUE"Attempting to build package..."$NOR | |||
|
173 | ||||
|
174 | tools/release | |||
|
175 | ||||
|
176 | ||||
|
177 | echo $RED'$ shasum -a 256 dist/*' | |||
|
178 | shasum -a 256 dist/* | |||
|
179 | echo $NOR | |||
|
180 | ||||
|
181 | echo $BLUE"We are going to rebuild, node the hash above, and compare them to the rebuild"$NOR | |||
|
182 | echo $GREEN"Press enter to continue"$NOR | |||
|
183 | read | |||
|
184 | ||||
164 | echo |
|
185 | echo | |
165 | echo $BLUE"Attempting to build package..."$NOR |
|
186 | echo $BLUE"Attempting to build package..."$NOR | |
166 |
|
187 | |||
167 | tools/release |
|
188 | tools/release | |
168 |
|
189 | |||
169 | echo $RED |
|
190 | echo $RED"Check the shasum for SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" | |
170 | echo '$ shasum -a 256 dist/*' |
|
191 | echo $RED'$ shasum -a 256 dist/*' | |
171 | shasum -a 256 dist/* |
|
192 | shasum -a 256 dist/* | |
172 | echo $NOR |
|
193 | echo $NOR | |
173 |
|
194 |
General Comments 0
You need to be logged in to leave comments.
Login now