##// END OF EJS Templates
contrib: install Rust 1.46.0...
Gregory Szorc -
r46220:6a36e2d2 default
parent child Browse files
Show More
@@ -1,597 +1,597 b''
1 # linux.py - Linux specific automation functionality
1 # linux.py - Linux specific automation functionality
2 #
2 #
3 # Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com>
3 # Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 # no-check-code because Python 3 native.
8 # no-check-code because Python 3 native.
9
9
10 import os
10 import os
11 import pathlib
11 import pathlib
12 import shlex
12 import shlex
13 import subprocess
13 import subprocess
14 import tempfile
14 import tempfile
15
15
16 from .ssh import exec_command
16 from .ssh import exec_command
17
17
18
18
19 # Linux distributions that are supported.
19 # Linux distributions that are supported.
20 DISTROS = {
20 DISTROS = {
21 'debian9',
21 'debian9',
22 'debian10',
22 'debian10',
23 'ubuntu18.04',
23 'ubuntu18.04',
24 'ubuntu19.04',
24 'ubuntu19.04',
25 }
25 }
26
26
27 INSTALL_PYTHONS = r'''
27 INSTALL_PYTHONS = r'''
28 PYENV2_VERSIONS="2.7.17 pypy2.7-7.2.0"
28 PYENV2_VERSIONS="2.7.17 pypy2.7-7.2.0"
29 PYENV3_VERSIONS="3.5.10 3.6.12 3.7.9 3.8.6 pypy3.5-7.0.0 pypy3.6-7.3.0"
29 PYENV3_VERSIONS="3.5.10 3.6.12 3.7.9 3.8.6 pypy3.5-7.0.0 pypy3.6-7.3.0"
30
30
31 git clone https://github.com/pyenv/pyenv.git /hgdev/pyenv
31 git clone https://github.com/pyenv/pyenv.git /hgdev/pyenv
32 pushd /hgdev/pyenv
32 pushd /hgdev/pyenv
33 git checkout 8bf79eb44ce1ea0a8b36544eb3e8a694a3a6ed78
33 git checkout 8bf79eb44ce1ea0a8b36544eb3e8a694a3a6ed78
34 popd
34 popd
35
35
36 export PYENV_ROOT="/hgdev/pyenv"
36 export PYENV_ROOT="/hgdev/pyenv"
37 export PATH="$PYENV_ROOT/bin:$PATH"
37 export PATH="$PYENV_ROOT/bin:$PATH"
38
38
39 # pip 19.2.3.
39 # pip 19.2.3.
40 PIP_SHA256=57e3643ff19f018f8a00dfaa6b7e4620e3c1a7a2171fd218425366ec006b3bfe
40 PIP_SHA256=57e3643ff19f018f8a00dfaa6b7e4620e3c1a7a2171fd218425366ec006b3bfe
41 wget -O get-pip.py --progress dot:mega https://github.com/pypa/get-pip/raw/309a56c5fd94bd1134053a541cb4657a4e47e09d/get-pip.py
41 wget -O get-pip.py --progress dot:mega https://github.com/pypa/get-pip/raw/309a56c5fd94bd1134053a541cb4657a4e47e09d/get-pip.py
42 echo "${PIP_SHA256} get-pip.py" | sha256sum --check -
42 echo "${PIP_SHA256} get-pip.py" | sha256sum --check -
43
43
44 VIRTUALENV_SHA256=f78d81b62d3147396ac33fc9d77579ddc42cc2a98dd9ea38886f616b33bc7fb2
44 VIRTUALENV_SHA256=f78d81b62d3147396ac33fc9d77579ddc42cc2a98dd9ea38886f616b33bc7fb2
45 VIRTUALENV_TARBALL=virtualenv-16.7.5.tar.gz
45 VIRTUALENV_TARBALL=virtualenv-16.7.5.tar.gz
46 wget -O ${VIRTUALENV_TARBALL} --progress dot:mega https://files.pythonhosted.org/packages/66/f0/6867af06d2e2f511e4e1d7094ff663acdebc4f15d4a0cb0fed1007395124/${VIRTUALENV_TARBALL}
46 wget -O ${VIRTUALENV_TARBALL} --progress dot:mega https://files.pythonhosted.org/packages/66/f0/6867af06d2e2f511e4e1d7094ff663acdebc4f15d4a0cb0fed1007395124/${VIRTUALENV_TARBALL}
47 echo "${VIRTUALENV_SHA256} ${VIRTUALENV_TARBALL}" | sha256sum --check -
47 echo "${VIRTUALENV_SHA256} ${VIRTUALENV_TARBALL}" | sha256sum --check -
48
48
49 for v in ${PYENV2_VERSIONS}; do
49 for v in ${PYENV2_VERSIONS}; do
50 pyenv install -v ${v}
50 pyenv install -v ${v}
51 ${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
51 ${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
52 ${PYENV_ROOT}/versions/${v}/bin/pip install ${VIRTUALENV_TARBALL}
52 ${PYENV_ROOT}/versions/${v}/bin/pip install ${VIRTUALENV_TARBALL}
53 ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/requirements-py2.txt
53 ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/requirements-py2.txt
54 done
54 done
55
55
56 for v in ${PYENV3_VERSIONS}; do
56 for v in ${PYENV3_VERSIONS}; do
57 pyenv install -v ${v}
57 pyenv install -v ${v}
58 ${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
58 ${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
59 ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/requirements-py3.txt
59 ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/requirements-py3.txt
60 done
60 done
61
61
62 pyenv global ${PYENV2_VERSIONS} ${PYENV3_VERSIONS} system
62 pyenv global ${PYENV2_VERSIONS} ${PYENV3_VERSIONS} system
63 '''.lstrip().replace(
63 '''.lstrip().replace(
64 '\r\n', '\n'
64 '\r\n', '\n'
65 )
65 )
66
66
67
67
68 INSTALL_RUST = r'''
68 INSTALL_RUST = r'''
69 RUSTUP_INIT_SHA256=a46fe67199b7bcbbde2dcbc23ae08db6f29883e260e23899a88b9073effc9076
69 RUSTUP_INIT_SHA256=a46fe67199b7bcbbde2dcbc23ae08db6f29883e260e23899a88b9073effc9076
70 wget -O rustup-init --progress dot:mega https://static.rust-lang.org/rustup/archive/1.18.3/x86_64-unknown-linux-gnu/rustup-init
70 wget -O rustup-init --progress dot:mega https://static.rust-lang.org/rustup/archive/1.18.3/x86_64-unknown-linux-gnu/rustup-init
71 echo "${RUSTUP_INIT_SHA256} rustup-init" | sha256sum --check -
71 echo "${RUSTUP_INIT_SHA256} rustup-init" | sha256sum --check -
72
72
73 chmod +x rustup-init
73 chmod +x rustup-init
74 sudo -H -u hg -g hg ./rustup-init -y
74 sudo -H -u hg -g hg ./rustup-init -y
75 sudo -H -u hg -g hg /home/hg/.cargo/bin/rustup install 1.31.1 1.42.0
75 sudo -H -u hg -g hg /home/hg/.cargo/bin/rustup install 1.31.1 1.46.0
76 sudo -H -u hg -g hg /home/hg/.cargo/bin/rustup component add clippy
76 sudo -H -u hg -g hg /home/hg/.cargo/bin/rustup component add clippy
77
77
78 sudo -H -u hg -g hg /home/hg/.cargo/bin/cargo install --version 0.7.0 pyoxidizer
78 sudo -H -u hg -g hg /home/hg/.cargo/bin/cargo install --version 0.7.0 pyoxidizer
79 '''
79 '''
80
80
81
81
82 BOOTSTRAP_VIRTUALENV = r'''
82 BOOTSTRAP_VIRTUALENV = r'''
83 /usr/bin/virtualenv /hgdev/venv-bootstrap
83 /usr/bin/virtualenv /hgdev/venv-bootstrap
84
84
85 HG_SHA256=35fc8ba5e0379c1b3affa2757e83fb0509e8ac314cbd9f1fd133cf265d16e49f
85 HG_SHA256=35fc8ba5e0379c1b3affa2757e83fb0509e8ac314cbd9f1fd133cf265d16e49f
86 HG_TARBALL=mercurial-5.1.1.tar.gz
86 HG_TARBALL=mercurial-5.1.1.tar.gz
87
87
88 wget -O ${HG_TARBALL} --progress dot:mega https://www.mercurial-scm.org/release/${HG_TARBALL}
88 wget -O ${HG_TARBALL} --progress dot:mega https://www.mercurial-scm.org/release/${HG_TARBALL}
89 echo "${HG_SHA256} ${HG_TARBALL}" | sha256sum --check -
89 echo "${HG_SHA256} ${HG_TARBALL}" | sha256sum --check -
90
90
91 /hgdev/venv-bootstrap/bin/pip install ${HG_TARBALL}
91 /hgdev/venv-bootstrap/bin/pip install ${HG_TARBALL}
92 '''.lstrip().replace(
92 '''.lstrip().replace(
93 '\r\n', '\n'
93 '\r\n', '\n'
94 )
94 )
95
95
96
96
97 BOOTSTRAP_DEBIAN = (
97 BOOTSTRAP_DEBIAN = (
98 r'''
98 r'''
99 #!/bin/bash
99 #!/bin/bash
100
100
101 set -ex
101 set -ex
102
102
103 DISTRO=`grep DISTRIB_ID /etc/lsb-release | awk -F= '{{print $2}}'`
103 DISTRO=`grep DISTRIB_ID /etc/lsb-release | awk -F= '{{print $2}}'`
104 DEBIAN_VERSION=`cat /etc/debian_version`
104 DEBIAN_VERSION=`cat /etc/debian_version`
105 LSB_RELEASE=`lsb_release -cs`
105 LSB_RELEASE=`lsb_release -cs`
106
106
107 sudo /usr/sbin/groupadd hg
107 sudo /usr/sbin/groupadd hg
108 sudo /usr/sbin/groupadd docker
108 sudo /usr/sbin/groupadd docker
109 sudo /usr/sbin/useradd -g hg -G sudo,docker -d /home/hg -m -s /bin/bash hg
109 sudo /usr/sbin/useradd -g hg -G sudo,docker -d /home/hg -m -s /bin/bash hg
110 sudo mkdir /home/hg/.ssh
110 sudo mkdir /home/hg/.ssh
111 sudo cp ~/.ssh/authorized_keys /home/hg/.ssh/authorized_keys
111 sudo cp ~/.ssh/authorized_keys /home/hg/.ssh/authorized_keys
112 sudo chown -R hg:hg /home/hg/.ssh
112 sudo chown -R hg:hg /home/hg/.ssh
113 sudo chmod 700 /home/hg/.ssh
113 sudo chmod 700 /home/hg/.ssh
114 sudo chmod 600 /home/hg/.ssh/authorized_keys
114 sudo chmod 600 /home/hg/.ssh/authorized_keys
115
115
116 cat << EOF | sudo tee /etc/sudoers.d/90-hg
116 cat << EOF | sudo tee /etc/sudoers.d/90-hg
117 hg ALL=(ALL) NOPASSWD:ALL
117 hg ALL=(ALL) NOPASSWD:ALL
118 EOF
118 EOF
119
119
120 sudo apt-get update
120 sudo apt-get update
121 sudo DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade
121 sudo DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade
122
122
123 # Install packages necessary to set up Docker Apt repo.
123 # Install packages necessary to set up Docker Apt repo.
124 sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install --no-install-recommends \
124 sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install --no-install-recommends \
125 apt-transport-https \
125 apt-transport-https \
126 gnupg
126 gnupg
127
127
128 cat > docker-apt-key << EOF
128 cat > docker-apt-key << EOF
129 -----BEGIN PGP PUBLIC KEY BLOCK-----
129 -----BEGIN PGP PUBLIC KEY BLOCK-----
130
130
131 mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth
131 mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth
132 lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh
132 lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh
133 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq
133 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq
134 L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7
134 L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7
135 UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N
135 UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N
136 cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht
136 cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht
137 ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo
137 ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo
138 vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD
138 vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD
139 G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ
139 G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ
140 XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj
140 XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj
141 q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB
141 q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB
142 tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3
142 tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3
143 BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO
143 BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO
144 v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd
144 v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd
145 tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk
145 tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk
146 jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m
146 jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m
147 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P
147 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P
148 XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc
148 XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc
149 FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8
149 FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8
150 g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm
150 g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm
151 ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh
151 ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh
152 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5
152 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5
153 G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW
153 G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW
154 FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB
154 FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB
155 EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF
155 EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF
156 M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx
156 M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx
157 Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu
157 Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu
158 w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk
158 w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk
159 z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8
159 z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8
160 eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb
160 eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb
161 VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa
161 VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa
162 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X
162 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X
163 zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ
163 zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ
164 pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7
164 pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7
165 ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ
165 ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ
166 BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY
166 BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY
167 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp
167 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp
168 YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI
168 YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI
169 mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES
169 mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES
170 KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7
170 KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7
171 JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ
171 JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ
172 cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0
172 cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0
173 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5
173 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5
174 U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z
174 U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z
175 VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f
175 VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f
176 irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk
176 irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk
177 SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz
177 SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz
178 QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W
178 QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W
179 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw
179 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw
180 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe
180 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe
181 dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y
181 dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y
182 Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR
182 Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR
183 H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh
183 H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh
184 /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ
184 /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ
185 M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S
185 M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S
186 xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O
186 xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O
187 jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG
187 jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG
188 YT90qFF93M3v01BbxP+EIY2/9tiIPbrd
188 YT90qFF93M3v01BbxP+EIY2/9tiIPbrd
189 =0YYh
189 =0YYh
190 -----END PGP PUBLIC KEY BLOCK-----
190 -----END PGP PUBLIC KEY BLOCK-----
191 EOF
191 EOF
192
192
193 sudo apt-key add docker-apt-key
193 sudo apt-key add docker-apt-key
194
194
195 if [ "$LSB_RELEASE" = "stretch" ]; then
195 if [ "$LSB_RELEASE" = "stretch" ]; then
196 cat << EOF | sudo tee -a /etc/apt/sources.list
196 cat << EOF | sudo tee -a /etc/apt/sources.list
197 # Need backports for clang-format-6.0
197 # Need backports for clang-format-6.0
198 deb http://deb.debian.org/debian stretch-backports main
198 deb http://deb.debian.org/debian stretch-backports main
199 EOF
199 EOF
200 fi
200 fi
201
201
202 if [ "$LSB_RELEASE" = "stretch" -o "$LSB_RELEASE" = "buster" ]; then
202 if [ "$LSB_RELEASE" = "stretch" -o "$LSB_RELEASE" = "buster" ]; then
203 cat << EOF | sudo tee -a /etc/apt/sources.list
203 cat << EOF | sudo tee -a /etc/apt/sources.list
204 # Sources are useful if we want to compile things locally.
204 # Sources are useful if we want to compile things locally.
205 deb-src http://deb.debian.org/debian $LSB_RELEASE main
205 deb-src http://deb.debian.org/debian $LSB_RELEASE main
206 deb-src http://security.debian.org/debian-security $LSB_RELEASE/updates main
206 deb-src http://security.debian.org/debian-security $LSB_RELEASE/updates main
207 deb-src http://deb.debian.org/debian $LSB_RELEASE-updates main
207 deb-src http://deb.debian.org/debian $LSB_RELEASE-updates main
208 deb-src http://deb.debian.org/debian $LSB_RELEASE-backports main
208 deb-src http://deb.debian.org/debian $LSB_RELEASE-backports main
209
209
210 deb [arch=amd64] https://download.docker.com/linux/debian $LSB_RELEASE stable
210 deb [arch=amd64] https://download.docker.com/linux/debian $LSB_RELEASE stable
211 EOF
211 EOF
212
212
213 elif [ "$DISTRO" = "Ubuntu" ]; then
213 elif [ "$DISTRO" = "Ubuntu" ]; then
214 cat << EOF | sudo tee -a /etc/apt/sources.list
214 cat << EOF | sudo tee -a /etc/apt/sources.list
215 deb [arch=amd64] https://download.docker.com/linux/ubuntu $LSB_RELEASE stable
215 deb [arch=amd64] https://download.docker.com/linux/ubuntu $LSB_RELEASE stable
216 EOF
216 EOF
217
217
218 fi
218 fi
219
219
220 sudo apt-get update
220 sudo apt-get update
221
221
222 PACKAGES="\
222 PACKAGES="\
223 awscli \
223 awscli \
224 btrfs-progs \
224 btrfs-progs \
225 build-essential \
225 build-essential \
226 bzr \
226 bzr \
227 clang-format-6.0 \
227 clang-format-6.0 \
228 cvs \
228 cvs \
229 darcs \
229 darcs \
230 debhelper \
230 debhelper \
231 devscripts \
231 devscripts \
232 docker-ce \
232 docker-ce \
233 dpkg-dev \
233 dpkg-dev \
234 dstat \
234 dstat \
235 emacs \
235 emacs \
236 gettext \
236 gettext \
237 git \
237 git \
238 htop \
238 htop \
239 iotop \
239 iotop \
240 jfsutils \
240 jfsutils \
241 libbz2-dev \
241 libbz2-dev \
242 libexpat1-dev \
242 libexpat1-dev \
243 libffi-dev \
243 libffi-dev \
244 libgdbm-dev \
244 libgdbm-dev \
245 liblzma-dev \
245 liblzma-dev \
246 libncurses5-dev \
246 libncurses5-dev \
247 libnss3-dev \
247 libnss3-dev \
248 libreadline-dev \
248 libreadline-dev \
249 libsqlite3-dev \
249 libsqlite3-dev \
250 libssl-dev \
250 libssl-dev \
251 netbase \
251 netbase \
252 ntfs-3g \
252 ntfs-3g \
253 nvme-cli \
253 nvme-cli \
254 pyflakes \
254 pyflakes \
255 pyflakes3 \
255 pyflakes3 \
256 pylint \
256 pylint \
257 pylint3 \
257 pylint3 \
258 python-all-dev \
258 python-all-dev \
259 python-dev \
259 python-dev \
260 python-docutils \
260 python-docutils \
261 python-fuzzywuzzy \
261 python-fuzzywuzzy \
262 python-pygments \
262 python-pygments \
263 python-subversion \
263 python-subversion \
264 python-vcr \
264 python-vcr \
265 python3-boto3 \
265 python3-boto3 \
266 python3-dev \
266 python3-dev \
267 python3-docutils \
267 python3-docutils \
268 python3-fuzzywuzzy \
268 python3-fuzzywuzzy \
269 python3-pygments \
269 python3-pygments \
270 python3-vcr \
270 python3-vcr \
271 python3-venv \
271 python3-venv \
272 rsync \
272 rsync \
273 sqlite3 \
273 sqlite3 \
274 subversion \
274 subversion \
275 tcl-dev \
275 tcl-dev \
276 tk-dev \
276 tk-dev \
277 tla \
277 tla \
278 unzip \
278 unzip \
279 uuid-dev \
279 uuid-dev \
280 vim \
280 vim \
281 virtualenv \
281 virtualenv \
282 wget \
282 wget \
283 xfsprogs \
283 xfsprogs \
284 zip \
284 zip \
285 zlib1g-dev"
285 zlib1g-dev"
286
286
287 if [ "LSB_RELEASE" = "stretch" ]; then
287 if [ "LSB_RELEASE" = "stretch" ]; then
288 PACKAGES="$PACKAGES linux-perf"
288 PACKAGES="$PACKAGES linux-perf"
289 elif [ "$DISTRO" = "Ubuntu" ]; then
289 elif [ "$DISTRO" = "Ubuntu" ]; then
290 PACKAGES="$PACKAGES linux-tools-common"
290 PACKAGES="$PACKAGES linux-tools-common"
291 fi
291 fi
292
292
293 # Monotone only available in older releases.
293 # Monotone only available in older releases.
294 if [ "$LSB_RELEASE" = "stretch" -o "$LSB_RELEASE" = "xenial" ]; then
294 if [ "$LSB_RELEASE" = "stretch" -o "$LSB_RELEASE" = "xenial" ]; then
295 PACKAGES="$PACKAGES monotone"
295 PACKAGES="$PACKAGES monotone"
296 fi
296 fi
297
297
298 sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install --no-install-recommends $PACKAGES
298 sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install --no-install-recommends $PACKAGES
299
299
300 # Create clang-format symlink so test harness finds it.
300 # Create clang-format symlink so test harness finds it.
301 sudo update-alternatives --install /usr/bin/clang-format clang-format \
301 sudo update-alternatives --install /usr/bin/clang-format clang-format \
302 /usr/bin/clang-format-6.0 1000
302 /usr/bin/clang-format-6.0 1000
303
303
304 sudo mkdir /hgdev
304 sudo mkdir /hgdev
305 # Will be normalized to hg:hg later.
305 # Will be normalized to hg:hg later.
306 sudo chown `whoami` /hgdev
306 sudo chown `whoami` /hgdev
307
307
308 {install_rust}
308 {install_rust}
309
309
310 cp requirements-py2.txt /hgdev/requirements-py2.txt
310 cp requirements-py2.txt /hgdev/requirements-py2.txt
311 cp requirements-py3.txt /hgdev/requirements-py3.txt
311 cp requirements-py3.txt /hgdev/requirements-py3.txt
312
312
313 # Disable the pip version check because it uses the network and can
313 # Disable the pip version check because it uses the network and can
314 # be annoying.
314 # be annoying.
315 cat << EOF | sudo tee -a /etc/pip.conf
315 cat << EOF | sudo tee -a /etc/pip.conf
316 [global]
316 [global]
317 disable-pip-version-check = True
317 disable-pip-version-check = True
318 EOF
318 EOF
319
319
320 {install_pythons}
320 {install_pythons}
321 {bootstrap_virtualenv}
321 {bootstrap_virtualenv}
322
322
323 /hgdev/venv-bootstrap/bin/hg clone https://www.mercurial-scm.org/repo/hg /hgdev/src
323 /hgdev/venv-bootstrap/bin/hg clone https://www.mercurial-scm.org/repo/hg /hgdev/src
324
324
325 # Mark the repo as non-publishing.
325 # Mark the repo as non-publishing.
326 cat >> /hgdev/src/.hg/hgrc << EOF
326 cat >> /hgdev/src/.hg/hgrc << EOF
327 [phases]
327 [phases]
328 publish = false
328 publish = false
329 EOF
329 EOF
330
330
331 sudo chown -R hg:hg /hgdev
331 sudo chown -R hg:hg /hgdev
332 '''.lstrip()
332 '''.lstrip()
333 .format(
333 .format(
334 install_rust=INSTALL_RUST,
334 install_rust=INSTALL_RUST,
335 install_pythons=INSTALL_PYTHONS,
335 install_pythons=INSTALL_PYTHONS,
336 bootstrap_virtualenv=BOOTSTRAP_VIRTUALENV,
336 bootstrap_virtualenv=BOOTSTRAP_VIRTUALENV,
337 )
337 )
338 .replace('\r\n', '\n')
338 .replace('\r\n', '\n')
339 )
339 )
340
340
341
341
342 # Prepares /hgdev for operations.
342 # Prepares /hgdev for operations.
343 PREPARE_HGDEV = '''
343 PREPARE_HGDEV = '''
344 #!/bin/bash
344 #!/bin/bash
345
345
346 set -e
346 set -e
347
347
348 FS=$1
348 FS=$1
349
349
350 ensure_device() {
350 ensure_device() {
351 if [ -z "${DEVICE}" ]; then
351 if [ -z "${DEVICE}" ]; then
352 echo "could not find block device to format"
352 echo "could not find block device to format"
353 exit 1
353 exit 1
354 fi
354 fi
355 }
355 }
356
356
357 # Determine device to partition for extra filesystem.
357 # Determine device to partition for extra filesystem.
358 # If only 1 volume is present, it will be the root volume and
358 # If only 1 volume is present, it will be the root volume and
359 # should be /dev/nvme0. If multiple volumes are present, the
359 # should be /dev/nvme0. If multiple volumes are present, the
360 # root volume could be nvme0 or nvme1. Use whichever one doesn't have
360 # root volume could be nvme0 or nvme1. Use whichever one doesn't have
361 # a partition.
361 # a partition.
362 if [ -e /dev/nvme1n1 ]; then
362 if [ -e /dev/nvme1n1 ]; then
363 if [ -e /dev/nvme0n1p1 ]; then
363 if [ -e /dev/nvme0n1p1 ]; then
364 DEVICE=/dev/nvme1n1
364 DEVICE=/dev/nvme1n1
365 else
365 else
366 DEVICE=/dev/nvme0n1
366 DEVICE=/dev/nvme0n1
367 fi
367 fi
368 else
368 else
369 DEVICE=
369 DEVICE=
370 fi
370 fi
371
371
372 sudo mkdir /hgwork
372 sudo mkdir /hgwork
373
373
374 if [ "${FS}" != "default" -a "${FS}" != "tmpfs" ]; then
374 if [ "${FS}" != "default" -a "${FS}" != "tmpfs" ]; then
375 ensure_device
375 ensure_device
376 echo "creating ${FS} filesystem on ${DEVICE}"
376 echo "creating ${FS} filesystem on ${DEVICE}"
377 fi
377 fi
378
378
379 if [ "${FS}" = "default" ]; then
379 if [ "${FS}" = "default" ]; then
380 :
380 :
381
381
382 elif [ "${FS}" = "btrfs" ]; then
382 elif [ "${FS}" = "btrfs" ]; then
383 sudo mkfs.btrfs ${DEVICE}
383 sudo mkfs.btrfs ${DEVICE}
384 sudo mount ${DEVICE} /hgwork
384 sudo mount ${DEVICE} /hgwork
385
385
386 elif [ "${FS}" = "ext3" ]; then
386 elif [ "${FS}" = "ext3" ]; then
387 # lazy_journal_init speeds up filesystem creation at the expense of
387 # lazy_journal_init speeds up filesystem creation at the expense of
388 # integrity if things crash. We are an ephemeral instance, so we don't
388 # integrity if things crash. We are an ephemeral instance, so we don't
389 # care about integrity.
389 # care about integrity.
390 sudo mkfs.ext3 -E lazy_journal_init=1 ${DEVICE}
390 sudo mkfs.ext3 -E lazy_journal_init=1 ${DEVICE}
391 sudo mount ${DEVICE} /hgwork
391 sudo mount ${DEVICE} /hgwork
392
392
393 elif [ "${FS}" = "ext4" ]; then
393 elif [ "${FS}" = "ext4" ]; then
394 sudo mkfs.ext4 -E lazy_journal_init=1 ${DEVICE}
394 sudo mkfs.ext4 -E lazy_journal_init=1 ${DEVICE}
395 sudo mount ${DEVICE} /hgwork
395 sudo mount ${DEVICE} /hgwork
396
396
397 elif [ "${FS}" = "jfs" ]; then
397 elif [ "${FS}" = "jfs" ]; then
398 sudo mkfs.jfs ${DEVICE}
398 sudo mkfs.jfs ${DEVICE}
399 sudo mount ${DEVICE} /hgwork
399 sudo mount ${DEVICE} /hgwork
400
400
401 elif [ "${FS}" = "tmpfs" ]; then
401 elif [ "${FS}" = "tmpfs" ]; then
402 echo "creating tmpfs volume in /hgwork"
402 echo "creating tmpfs volume in /hgwork"
403 sudo mount -t tmpfs -o size=1024M tmpfs /hgwork
403 sudo mount -t tmpfs -o size=1024M tmpfs /hgwork
404
404
405 elif [ "${FS}" = "xfs" ]; then
405 elif [ "${FS}" = "xfs" ]; then
406 sudo mkfs.xfs ${DEVICE}
406 sudo mkfs.xfs ${DEVICE}
407 sudo mount ${DEVICE} /hgwork
407 sudo mount ${DEVICE} /hgwork
408
408
409 else
409 else
410 echo "unsupported filesystem: ${FS}"
410 echo "unsupported filesystem: ${FS}"
411 exit 1
411 exit 1
412 fi
412 fi
413
413
414 echo "/hgwork ready"
414 echo "/hgwork ready"
415
415
416 sudo chown hg:hg /hgwork
416 sudo chown hg:hg /hgwork
417 mkdir /hgwork/tmp
417 mkdir /hgwork/tmp
418 chown hg:hg /hgwork/tmp
418 chown hg:hg /hgwork/tmp
419
419
420 rsync -a /hgdev/src /hgwork/
420 rsync -a /hgdev/src /hgwork/
421 '''.lstrip().replace(
421 '''.lstrip().replace(
422 '\r\n', '\n'
422 '\r\n', '\n'
423 )
423 )
424
424
425
425
426 HG_UPDATE_CLEAN = '''
426 HG_UPDATE_CLEAN = '''
427 set -ex
427 set -ex
428
428
429 HG=/hgdev/venv-bootstrap/bin/hg
429 HG=/hgdev/venv-bootstrap/bin/hg
430
430
431 cd /hgwork/src
431 cd /hgwork/src
432 ${HG} --config extensions.purge= purge --all
432 ${HG} --config extensions.purge= purge --all
433 ${HG} update -C $1
433 ${HG} update -C $1
434 ${HG} log -r .
434 ${HG} log -r .
435 '''.lstrip().replace(
435 '''.lstrip().replace(
436 '\r\n', '\n'
436 '\r\n', '\n'
437 )
437 )
438
438
439
439
440 def prepare_exec_environment(ssh_client, filesystem='default'):
440 def prepare_exec_environment(ssh_client, filesystem='default'):
441 """Prepare an EC2 instance to execute things.
441 """Prepare an EC2 instance to execute things.
442
442
443 The AMI has an ``/hgdev`` bootstrapped with various Python installs
443 The AMI has an ``/hgdev`` bootstrapped with various Python installs
444 and a clone of the Mercurial repo.
444 and a clone of the Mercurial repo.
445
445
446 In EC2, EBS volumes launched from snapshots have wonky performance behavior.
446 In EC2, EBS volumes launched from snapshots have wonky performance behavior.
447 Notably, blocks have to be copied on first access, which makes volume
447 Notably, blocks have to be copied on first access, which makes volume
448 I/O extremely slow on fresh volumes.
448 I/O extremely slow on fresh volumes.
449
449
450 Furthermore, we may want to run operations, tests, etc on alternative
450 Furthermore, we may want to run operations, tests, etc on alternative
451 filesystems so we examine behavior on different filesystems.
451 filesystems so we examine behavior on different filesystems.
452
452
453 This function is used to facilitate executing operations on alternate
453 This function is used to facilitate executing operations on alternate
454 volumes.
454 volumes.
455 """
455 """
456 sftp = ssh_client.open_sftp()
456 sftp = ssh_client.open_sftp()
457
457
458 with sftp.open('/hgdev/prepare-hgdev', 'wb') as fh:
458 with sftp.open('/hgdev/prepare-hgdev', 'wb') as fh:
459 fh.write(PREPARE_HGDEV)
459 fh.write(PREPARE_HGDEV)
460 fh.chmod(0o0777)
460 fh.chmod(0o0777)
461
461
462 command = 'sudo /hgdev/prepare-hgdev %s' % filesystem
462 command = 'sudo /hgdev/prepare-hgdev %s' % filesystem
463 chan, stdin, stdout = exec_command(ssh_client, command)
463 chan, stdin, stdout = exec_command(ssh_client, command)
464 stdin.close()
464 stdin.close()
465
465
466 for line in stdout:
466 for line in stdout:
467 print(line, end='')
467 print(line, end='')
468
468
469 res = chan.recv_exit_status()
469 res = chan.recv_exit_status()
470
470
471 if res:
471 if res:
472 raise Exception('non-0 exit code updating working directory; %d' % res)
472 raise Exception('non-0 exit code updating working directory; %d' % res)
473
473
474
474
475 def synchronize_hg(
475 def synchronize_hg(
476 source_path: pathlib.Path, ec2_instance, revision: str = None
476 source_path: pathlib.Path, ec2_instance, revision: str = None
477 ):
477 ):
478 """Synchronize a local Mercurial source path to remote EC2 instance."""
478 """Synchronize a local Mercurial source path to remote EC2 instance."""
479
479
480 with tempfile.TemporaryDirectory() as temp_dir:
480 with tempfile.TemporaryDirectory() as temp_dir:
481 temp_dir = pathlib.Path(temp_dir)
481 temp_dir = pathlib.Path(temp_dir)
482
482
483 ssh_dir = temp_dir / '.ssh'
483 ssh_dir = temp_dir / '.ssh'
484 ssh_dir.mkdir()
484 ssh_dir.mkdir()
485 ssh_dir.chmod(0o0700)
485 ssh_dir.chmod(0o0700)
486
486
487 public_ip = ec2_instance.public_ip_address
487 public_ip = ec2_instance.public_ip_address
488
488
489 ssh_config = ssh_dir / 'config'
489 ssh_config = ssh_dir / 'config'
490
490
491 with ssh_config.open('w', encoding='utf-8') as fh:
491 with ssh_config.open('w', encoding='utf-8') as fh:
492 fh.write('Host %s\n' % public_ip)
492 fh.write('Host %s\n' % public_ip)
493 fh.write(' User hg\n')
493 fh.write(' User hg\n')
494 fh.write(' StrictHostKeyChecking no\n')
494 fh.write(' StrictHostKeyChecking no\n')
495 fh.write(' UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts'))
495 fh.write(' UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts'))
496 fh.write(' IdentityFile %s\n' % ec2_instance.ssh_private_key_path)
496 fh.write(' IdentityFile %s\n' % ec2_instance.ssh_private_key_path)
497
497
498 if not (source_path / '.hg').is_dir():
498 if not (source_path / '.hg').is_dir():
499 raise Exception(
499 raise Exception(
500 '%s is not a Mercurial repository; synchronization '
500 '%s is not a Mercurial repository; synchronization '
501 'not yet supported' % source_path
501 'not yet supported' % source_path
502 )
502 )
503
503
504 env = dict(os.environ)
504 env = dict(os.environ)
505 env['HGPLAIN'] = '1'
505 env['HGPLAIN'] = '1'
506 env['HGENCODING'] = 'utf-8'
506 env['HGENCODING'] = 'utf-8'
507
507
508 hg_bin = source_path / 'hg'
508 hg_bin = source_path / 'hg'
509
509
510 res = subprocess.run(
510 res = subprocess.run(
511 ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
511 ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
512 cwd=str(source_path),
512 cwd=str(source_path),
513 env=env,
513 env=env,
514 check=True,
514 check=True,
515 capture_output=True,
515 capture_output=True,
516 )
516 )
517
517
518 full_revision = res.stdout.decode('ascii')
518 full_revision = res.stdout.decode('ascii')
519
519
520 args = [
520 args = [
521 'python2.7',
521 'python2.7',
522 str(hg_bin),
522 str(hg_bin),
523 '--config',
523 '--config',
524 'ui.ssh=ssh -F %s' % ssh_config,
524 'ui.ssh=ssh -F %s' % ssh_config,
525 '--config',
525 '--config',
526 'ui.remotecmd=/hgdev/venv-bootstrap/bin/hg',
526 'ui.remotecmd=/hgdev/venv-bootstrap/bin/hg',
527 # Also ensure .hgtags changes are present so auto version
527 # Also ensure .hgtags changes are present so auto version
528 # calculation works.
528 # calculation works.
529 'push',
529 'push',
530 '-f',
530 '-f',
531 '-r',
531 '-r',
532 full_revision,
532 full_revision,
533 '-r',
533 '-r',
534 'file(.hgtags)',
534 'file(.hgtags)',
535 'ssh://%s//hgwork/src' % public_ip,
535 'ssh://%s//hgwork/src' % public_ip,
536 ]
536 ]
537
537
538 res = subprocess.run(args, cwd=str(source_path), env=env)
538 res = subprocess.run(args, cwd=str(source_path), env=env)
539
539
540 # Allow 1 (no-op) to not trigger error.
540 # Allow 1 (no-op) to not trigger error.
541 if res.returncode not in (0, 1):
541 if res.returncode not in (0, 1):
542 res.check_returncode()
542 res.check_returncode()
543
543
544 # TODO support synchronizing dirty working directory.
544 # TODO support synchronizing dirty working directory.
545
545
546 sftp = ec2_instance.ssh_client.open_sftp()
546 sftp = ec2_instance.ssh_client.open_sftp()
547
547
548 with sftp.open('/hgdev/hgup', 'wb') as fh:
548 with sftp.open('/hgdev/hgup', 'wb') as fh:
549 fh.write(HG_UPDATE_CLEAN)
549 fh.write(HG_UPDATE_CLEAN)
550 fh.chmod(0o0700)
550 fh.chmod(0o0700)
551
551
552 chan, stdin, stdout = exec_command(
552 chan, stdin, stdout = exec_command(
553 ec2_instance.ssh_client, '/hgdev/hgup %s' % full_revision
553 ec2_instance.ssh_client, '/hgdev/hgup %s' % full_revision
554 )
554 )
555 stdin.close()
555 stdin.close()
556
556
557 for line in stdout:
557 for line in stdout:
558 print(line, end='')
558 print(line, end='')
559
559
560 res = chan.recv_exit_status()
560 res = chan.recv_exit_status()
561
561
562 if res:
562 if res:
563 raise Exception(
563 raise Exception(
564 'non-0 exit code updating working directory; %d' % res
564 'non-0 exit code updating working directory; %d' % res
565 )
565 )
566
566
567
567
568 def run_tests(ssh_client, python_version, test_flags=None):
568 def run_tests(ssh_client, python_version, test_flags=None):
569 """Run tests on a remote Linux machine via an SSH client."""
569 """Run tests on a remote Linux machine via an SSH client."""
570 test_flags = test_flags or []
570 test_flags = test_flags or []
571
571
572 print('running tests')
572 print('running tests')
573
573
574 if python_version == 'system2':
574 if python_version == 'system2':
575 python = '/usr/bin/python2'
575 python = '/usr/bin/python2'
576 elif python_version == 'system3':
576 elif python_version == 'system3':
577 python = '/usr/bin/python3'
577 python = '/usr/bin/python3'
578 elif python_version.startswith('pypy'):
578 elif python_version.startswith('pypy'):
579 python = '/hgdev/pyenv/shims/%s' % python_version
579 python = '/hgdev/pyenv/shims/%s' % python_version
580 else:
580 else:
581 python = '/hgdev/pyenv/shims/python%s' % python_version
581 python = '/hgdev/pyenv/shims/python%s' % python_version
582
582
583 test_flags = ' '.join(shlex.quote(a) for a in test_flags)
583 test_flags = ' '.join(shlex.quote(a) for a in test_flags)
584
584
585 command = (
585 command = (
586 '/bin/sh -c "export TMPDIR=/hgwork/tmp; '
586 '/bin/sh -c "export TMPDIR=/hgwork/tmp; '
587 'cd /hgwork/src/tests && %s run-tests.py %s"' % (python, test_flags)
587 'cd /hgwork/src/tests && %s run-tests.py %s"' % (python, test_flags)
588 )
588 )
589
589
590 chan, stdin, stdout = exec_command(ssh_client, command)
590 chan, stdin, stdout = exec_command(ssh_client, command)
591
591
592 stdin.close()
592 stdin.close()
593
593
594 for line in stdout:
594 for line in stdout:
595 print(line, end='')
595 print(line, end='')
596
596
597 return chan.recv_exit_status()
597 return chan.recv_exit_status()
@@ -1,220 +1,220 b''
1 # install-dependencies.ps1 - Install Windows dependencies for building Mercurial
1 # install-dependencies.ps1 - Install Windows dependencies for building Mercurial
2 #
2 #
3 # Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com>
3 # Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com>
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 # This script can be used to bootstrap a Mercurial build environment on
8 # This script can be used to bootstrap a Mercurial build environment on
9 # Windows.
9 # Windows.
10 #
10 #
11 # The script makes a lot of assumptions about how things should work.
11 # The script makes a lot of assumptions about how things should work.
12 # For example, the install location of Python is hardcoded to c:\hgdev\*.
12 # For example, the install location of Python is hardcoded to c:\hgdev\*.
13 #
13 #
14 # The script should be executed from a PowerShell with elevated privileges
14 # The script should be executed from a PowerShell with elevated privileges
15 # if you don't want to see a UAC prompt for various installers.
15 # if you don't want to see a UAC prompt for various installers.
16 #
16 #
17 # The script is tested on Windows 10 and Windows Server 2019 (in EC2).
17 # The script is tested on Windows 10 and Windows Server 2019 (in EC2).
18
18
19 $VS_BUILD_TOOLS_URL = "https://download.visualstudio.microsoft.com/download/pr/a1603c02-8a66-4b83-b821-811e3610a7c4/aa2db8bb39e0cbd23e9940d8951e0bc3/vs_buildtools.exe"
19 $VS_BUILD_TOOLS_URL = "https://download.visualstudio.microsoft.com/download/pr/a1603c02-8a66-4b83-b821-811e3610a7c4/aa2db8bb39e0cbd23e9940d8951e0bc3/vs_buildtools.exe"
20 $VS_BUILD_TOOLS_SHA256 = "911E292B8E6E5F46CBC17003BDCD2D27A70E616E8D5E6E69D5D489A605CAA139"
20 $VS_BUILD_TOOLS_SHA256 = "911E292B8E6E5F46CBC17003BDCD2D27A70E616E8D5E6E69D5D489A605CAA139"
21
21
22 $VC9_PYTHON_URL = "https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi"
22 $VC9_PYTHON_URL = "https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi"
23 $VC9_PYTHON_SHA256 = "070474db76a2e625513a5835df4595df9324d820f9cc97eab2a596dcbc2f5cbf"
23 $VC9_PYTHON_SHA256 = "070474db76a2e625513a5835df4595df9324d820f9cc97eab2a596dcbc2f5cbf"
24
24
25 $PYTHON27_x64_URL = "https://www.python.org/ftp/python/2.7.18/python-2.7.18.amd64.msi"
25 $PYTHON27_x64_URL = "https://www.python.org/ftp/python/2.7.18/python-2.7.18.amd64.msi"
26 $PYTHON27_x64_SHA256 = "b74a3afa1e0bf2a6fc566a7b70d15c9bfabba3756fb077797d16fffa27800c05"
26 $PYTHON27_x64_SHA256 = "b74a3afa1e0bf2a6fc566a7b70d15c9bfabba3756fb077797d16fffa27800c05"
27 $PYTHON27_X86_URL = "https://www.python.org/ftp/python/2.7.18/python-2.7.18.msi"
27 $PYTHON27_X86_URL = "https://www.python.org/ftp/python/2.7.18/python-2.7.18.msi"
28 $PYTHON27_X86_SHA256 = "d901802e90026e9bad76b8a81f8dd7e43c7d7e8269d9281c9e9df7a9c40480a9"
28 $PYTHON27_X86_SHA256 = "d901802e90026e9bad76b8a81f8dd7e43c7d7e8269d9281c9e9df7a9c40480a9"
29
29
30 $PYTHON35_x86_URL = "https://www.python.org/ftp/python/3.5.4/python-3.5.4.exe"
30 $PYTHON35_x86_URL = "https://www.python.org/ftp/python/3.5.4/python-3.5.4.exe"
31 $PYTHON35_x86_SHA256 = "F27C2D67FD9688E4970F3BFF799BB9D722A0D6C2C13B04848E1F7D620B524B0E"
31 $PYTHON35_x86_SHA256 = "F27C2D67FD9688E4970F3BFF799BB9D722A0D6C2C13B04848E1F7D620B524B0E"
32 $PYTHON35_x64_URL = "https://www.python.org/ftp/python/3.5.4/python-3.5.4-amd64.exe"
32 $PYTHON35_x64_URL = "https://www.python.org/ftp/python/3.5.4/python-3.5.4-amd64.exe"
33 $PYTHON35_x64_SHA256 = "9B7741CC32357573A77D2EE64987717E527628C38FD7EAF3E2AACA853D45A1EE"
33 $PYTHON35_x64_SHA256 = "9B7741CC32357573A77D2EE64987717E527628C38FD7EAF3E2AACA853D45A1EE"
34
34
35 $PYTHON36_x86_URL = "https://www.python.org/ftp/python/3.6.8/python-3.6.8.exe"
35 $PYTHON36_x86_URL = "https://www.python.org/ftp/python/3.6.8/python-3.6.8.exe"
36 $PYTHON36_x86_SHA256 = "89871D432BC06E4630D7B64CB1A8451E53C80E68DE29029976B12AAD7DBFA5A0"
36 $PYTHON36_x86_SHA256 = "89871D432BC06E4630D7B64CB1A8451E53C80E68DE29029976B12AAD7DBFA5A0"
37 $PYTHON36_x64_URL = "https://www.python.org/ftp/python/3.6.8/python-3.6.8-amd64.exe"
37 $PYTHON36_x64_URL = "https://www.python.org/ftp/python/3.6.8/python-3.6.8-amd64.exe"
38 $PYTHON36_x64_SHA256 = "96088A58B7C43BC83B84E6B67F15E8706C614023DD64F9A5A14E81FF824ADADC"
38 $PYTHON36_x64_SHA256 = "96088A58B7C43BC83B84E6B67F15E8706C614023DD64F9A5A14E81FF824ADADC"
39
39
40 $PYTHON37_x86_URL = "https://www.python.org/ftp/python/3.7.9/python-3.7.9.exe"
40 $PYTHON37_x86_URL = "https://www.python.org/ftp/python/3.7.9/python-3.7.9.exe"
41 $PYTHON37_x86_SHA256 = "769bb7c74ad1df6d7d74071cc16a984ff6182e4016e11b8949b93db487977220"
41 $PYTHON37_x86_SHA256 = "769bb7c74ad1df6d7d74071cc16a984ff6182e4016e11b8949b93db487977220"
42 $PYTHON37_X64_URL = "https://www.python.org/ftp/python/3.7.9/python-3.7.9-amd64.exe"
42 $PYTHON37_X64_URL = "https://www.python.org/ftp/python/3.7.9/python-3.7.9-amd64.exe"
43 $PYTHON37_x64_SHA256 = "e69ed52afb5a722e5c56f6c21d594e85c17cb29f12f18bb69751cf1714e0f987"
43 $PYTHON37_x64_SHA256 = "e69ed52afb5a722e5c56f6c21d594e85c17cb29f12f18bb69751cf1714e0f987"
44
44
45 $PYTHON38_x86_URL = "https://www.python.org/ftp/python/3.8.6/python-3.8.6.exe"
45 $PYTHON38_x86_URL = "https://www.python.org/ftp/python/3.8.6/python-3.8.6.exe"
46 $PYTHON38_x86_SHA256 = "287d5df01ff22ff09e6a487ae018603ee19eade71d462ec703850c96f1d5e8a0"
46 $PYTHON38_x86_SHA256 = "287d5df01ff22ff09e6a487ae018603ee19eade71d462ec703850c96f1d5e8a0"
47 $PYTHON38_x64_URL = "https://www.python.org/ftp/python/3.8.6/python-3.8.6-amd64.exe"
47 $PYTHON38_x64_URL = "https://www.python.org/ftp/python/3.8.6/python-3.8.6-amd64.exe"
48 $PYTHON38_x64_SHA256 = "328a257f189cb500606bb26ab0fbdd298ed0e05d8c36540a322a1744f489a0a0"
48 $PYTHON38_x64_SHA256 = "328a257f189cb500606bb26ab0fbdd298ed0e05d8c36540a322a1744f489a0a0"
49
49
50 # PIP 19.2.3.
50 # PIP 19.2.3.
51 $PIP_URL = "https://github.com/pypa/get-pip/raw/309a56c5fd94bd1134053a541cb4657a4e47e09d/get-pip.py"
51 $PIP_URL = "https://github.com/pypa/get-pip/raw/309a56c5fd94bd1134053a541cb4657a4e47e09d/get-pip.py"
52 $PIP_SHA256 = "57e3643ff19f018f8a00dfaa6b7e4620e3c1a7a2171fd218425366ec006b3bfe"
52 $PIP_SHA256 = "57e3643ff19f018f8a00dfaa6b7e4620e3c1a7a2171fd218425366ec006b3bfe"
53
53
54 $VIRTUALENV_URL = "https://files.pythonhosted.org/packages/66/f0/6867af06d2e2f511e4e1d7094ff663acdebc4f15d4a0cb0fed1007395124/virtualenv-16.7.5.tar.gz"
54 $VIRTUALENV_URL = "https://files.pythonhosted.org/packages/66/f0/6867af06d2e2f511e4e1d7094ff663acdebc4f15d4a0cb0fed1007395124/virtualenv-16.7.5.tar.gz"
55 $VIRTUALENV_SHA256 = "f78d81b62d3147396ac33fc9d77579ddc42cc2a98dd9ea38886f616b33bc7fb2"
55 $VIRTUALENV_SHA256 = "f78d81b62d3147396ac33fc9d77579ddc42cc2a98dd9ea38886f616b33bc7fb2"
56
56
57 $INNO_SETUP_URL = "http://files.jrsoftware.org/is/5/innosetup-5.6.1-unicode.exe"
57 $INNO_SETUP_URL = "http://files.jrsoftware.org/is/5/innosetup-5.6.1-unicode.exe"
58 $INNO_SETUP_SHA256 = "27D49E9BC769E9D1B214C153011978DB90DC01C2ACD1DDCD9ED7B3FE3B96B538"
58 $INNO_SETUP_SHA256 = "27D49E9BC769E9D1B214C153011978DB90DC01C2ACD1DDCD9ED7B3FE3B96B538"
59
59
60 $MINGW_BIN_URL = "https://osdn.net/frs/redir.php?m=constant&f=mingw%2F68260%2Fmingw-get-0.6.3-mingw32-pre-20170905-1-bin.zip"
60 $MINGW_BIN_URL = "https://osdn.net/frs/redir.php?m=constant&f=mingw%2F68260%2Fmingw-get-0.6.3-mingw32-pre-20170905-1-bin.zip"
61 $MINGW_BIN_SHA256 = "2AB8EFD7C7D1FC8EAF8B2FA4DA4EEF8F3E47768284C021599BC7435839A046DF"
61 $MINGW_BIN_SHA256 = "2AB8EFD7C7D1FC8EAF8B2FA4DA4EEF8F3E47768284C021599BC7435839A046DF"
62
62
63 $MERCURIAL_WHEEL_FILENAME = "mercurial-5.1.2-cp27-cp27m-win_amd64.whl"
63 $MERCURIAL_WHEEL_FILENAME = "mercurial-5.1.2-cp27-cp27m-win_amd64.whl"
64 $MERCURIAL_WHEEL_URL = "https://files.pythonhosted.org/packages/6d/47/e031e47f7fe9b16e4e3383da47e2b0a7eae6e603996bc67a03ec4fa1b3f4/$MERCURIAL_WHEEL_FILENAME"
64 $MERCURIAL_WHEEL_URL = "https://files.pythonhosted.org/packages/6d/47/e031e47f7fe9b16e4e3383da47e2b0a7eae6e603996bc67a03ec4fa1b3f4/$MERCURIAL_WHEEL_FILENAME"
65 $MERCURIAL_WHEEL_SHA256 = "1d18c7f6ca1456f0f62ee65c9a50c14cbba48ce6e924930cdb10537f5c9eaf5f"
65 $MERCURIAL_WHEEL_SHA256 = "1d18c7f6ca1456f0f62ee65c9a50c14cbba48ce6e924930cdb10537f5c9eaf5f"
66
66
67 $RUSTUP_INIT_URL = "https://static.rust-lang.org/rustup/archive/1.21.1/x86_64-pc-windows-gnu/rustup-init.exe"
67 $RUSTUP_INIT_URL = "https://static.rust-lang.org/rustup/archive/1.21.1/x86_64-pc-windows-gnu/rustup-init.exe"
68 $RUSTUP_INIT_SHA256 = "d17df34ba974b9b19cf5c75883a95475aa22ddc364591d75d174090d55711c72"
68 $RUSTUP_INIT_SHA256 = "d17df34ba974b9b19cf5c75883a95475aa22ddc364591d75d174090d55711c72"
69
69
70 # Writing progress slows down downloads substantially. So disable it.
70 # Writing progress slows down downloads substantially. So disable it.
71 $progressPreference = 'silentlyContinue'
71 $progressPreference = 'silentlyContinue'
72
72
73 function Secure-Download($url, $path, $sha256) {
73 function Secure-Download($url, $path, $sha256) {
74 if (Test-Path -Path $path) {
74 if (Test-Path -Path $path) {
75 Get-FileHash -Path $path -Algorithm SHA256 -OutVariable hash
75 Get-FileHash -Path $path -Algorithm SHA256 -OutVariable hash
76
76
77 if ($hash.Hash -eq $sha256) {
77 if ($hash.Hash -eq $sha256) {
78 Write-Output "SHA256 of $path verified as $sha256"
78 Write-Output "SHA256 of $path verified as $sha256"
79 return
79 return
80 }
80 }
81
81
82 Write-Output "hash mismatch on $path; downloading again"
82 Write-Output "hash mismatch on $path; downloading again"
83 }
83 }
84
84
85 Write-Output "downloading $url to $path"
85 Write-Output "downloading $url to $path"
86 Invoke-WebRequest -Uri $url -OutFile $path
86 Invoke-WebRequest -Uri $url -OutFile $path
87 Get-FileHash -Path $path -Algorithm SHA256 -OutVariable hash
87 Get-FileHash -Path $path -Algorithm SHA256 -OutVariable hash
88
88
89 if ($hash.Hash -ne $sha256) {
89 if ($hash.Hash -ne $sha256) {
90 Remove-Item -Path $path
90 Remove-Item -Path $path
91 throw "hash mismatch when downloading $url; got $($hash.Hash), expected $sha256"
91 throw "hash mismatch when downloading $url; got $($hash.Hash), expected $sha256"
92 }
92 }
93 }
93 }
94
94
95 function Invoke-Process($path, $arguments) {
95 function Invoke-Process($path, $arguments) {
96 $p = Start-Process -FilePath $path -ArgumentList $arguments -Wait -PassThru -WindowStyle Hidden
96 $p = Start-Process -FilePath $path -ArgumentList $arguments -Wait -PassThru -WindowStyle Hidden
97
97
98 if ($p.ExitCode -ne 0) {
98 if ($p.ExitCode -ne 0) {
99 throw "process exited non-0: $($p.ExitCode)"
99 throw "process exited non-0: $($p.ExitCode)"
100 }
100 }
101 }
101 }
102
102
103 function Install-Python3($name, $installer, $dest, $pip) {
103 function Install-Python3($name, $installer, $dest, $pip) {
104 Write-Output "installing $name"
104 Write-Output "installing $name"
105
105
106 # We hit this when running the script as part of Simple Systems Manager in
106 # We hit this when running the script as part of Simple Systems Manager in
107 # EC2. The Python 3 installer doesn't seem to like per-user installs
107 # EC2. The Python 3 installer doesn't seem to like per-user installs
108 # when running as the SYSTEM user. So enable global installs if executed in
108 # when running as the SYSTEM user. So enable global installs if executed in
109 # this mode.
109 # this mode.
110 if ($env:USERPROFILE -eq "C:\Windows\system32\config\systemprofile") {
110 if ($env:USERPROFILE -eq "C:\Windows\system32\config\systemprofile") {
111 Write-Output "running with SYSTEM account; installing for all users"
111 Write-Output "running with SYSTEM account; installing for all users"
112 $allusers = "1"
112 $allusers = "1"
113 }
113 }
114 else {
114 else {
115 $allusers = "0"
115 $allusers = "0"
116 }
116 }
117
117
118 Invoke-Process $installer "/quiet TargetDir=${dest} InstallAllUsers=${allusers} AssociateFiles=0 CompileAll=0 PrependPath=0 Include_doc=0 Include_launcher=0 InstallLauncherAllUsers=0 Include_pip=0 Include_test=0"
118 Invoke-Process $installer "/quiet TargetDir=${dest} InstallAllUsers=${allusers} AssociateFiles=0 CompileAll=0 PrependPath=0 Include_doc=0 Include_launcher=0 InstallLauncherAllUsers=0 Include_pip=0 Include_test=0"
119 Invoke-Process ${dest}\python.exe $pip
119 Invoke-Process ${dest}\python.exe $pip
120 }
120 }
121
121
122 function Install-Rust($prefix) {
122 function Install-Rust($prefix) {
123 Write-Output "installing Rust"
123 Write-Output "installing Rust"
124 $Env:RUSTUP_HOME = "${prefix}\rustup"
124 $Env:RUSTUP_HOME = "${prefix}\rustup"
125 $Env:CARGO_HOME = "${prefix}\cargo"
125 $Env:CARGO_HOME = "${prefix}\cargo"
126
126
127 Invoke-Process "${prefix}\assets\rustup-init.exe" "-y --default-host x86_64-pc-windows-msvc"
127 Invoke-Process "${prefix}\assets\rustup-init.exe" "-y --default-host x86_64-pc-windows-msvc"
128 Invoke-Process "${prefix}\cargo\bin\rustup.exe" "target add i686-pc-windows-msvc"
128 Invoke-Process "${prefix}\cargo\bin\rustup.exe" "target add i686-pc-windows-msvc"
129 Invoke-Process "${prefix}\cargo\bin\rustup.exe" "install 1.42.0"
129 Invoke-Process "${prefix}\cargo\bin\rustup.exe" "install 1.46.0"
130 Invoke-Process "${prefix}\cargo\bin\rustup.exe" "component add clippy"
130 Invoke-Process "${prefix}\cargo\bin\rustup.exe" "component add clippy"
131
131
132 # Install PyOxidizer for packaging.
132 # Install PyOxidizer for packaging.
133 Invoke-Process "${prefix}\cargo\bin\cargo.exe" "install --version 0.7.0 pyoxidizer"
133 Invoke-Process "${prefix}\cargo\bin\cargo.exe" "install --version 0.7.0 pyoxidizer"
134 }
134 }
135
135
136 function Install-Dependencies($prefix) {
136 function Install-Dependencies($prefix) {
137 if (!(Test-Path -Path $prefix\assets)) {
137 if (!(Test-Path -Path $prefix\assets)) {
138 New-Item -Path $prefix\assets -ItemType Directory
138 New-Item -Path $prefix\assets -ItemType Directory
139 }
139 }
140
140
141 $pip = "${prefix}\assets\get-pip.py"
141 $pip = "${prefix}\assets\get-pip.py"
142
142
143 Secure-Download $VC9_PYTHON_URL ${prefix}\assets\VCForPython27.msi $VC9_PYTHON_SHA256
143 Secure-Download $VC9_PYTHON_URL ${prefix}\assets\VCForPython27.msi $VC9_PYTHON_SHA256
144 Secure-Download $PYTHON27_x86_URL ${prefix}\assets\python27-x86.msi $PYTHON27_x86_SHA256
144 Secure-Download $PYTHON27_x86_URL ${prefix}\assets\python27-x86.msi $PYTHON27_x86_SHA256
145 Secure-Download $PYTHON27_x64_URL ${prefix}\assets\python27-x64.msi $PYTHON27_x64_SHA256
145 Secure-Download $PYTHON27_x64_URL ${prefix}\assets\python27-x64.msi $PYTHON27_x64_SHA256
146 Secure-Download $PYTHON35_x86_URL ${prefix}\assets\python35-x86.exe $PYTHON35_x86_SHA256
146 Secure-Download $PYTHON35_x86_URL ${prefix}\assets\python35-x86.exe $PYTHON35_x86_SHA256
147 Secure-Download $PYTHON35_x64_URL ${prefix}\assets\python35-x64.exe $PYTHON35_x64_SHA256
147 Secure-Download $PYTHON35_x64_URL ${prefix}\assets\python35-x64.exe $PYTHON35_x64_SHA256
148 Secure-Download $PYTHON36_x86_URL ${prefix}\assets\python36-x86.exe $PYTHON36_x86_SHA256
148 Secure-Download $PYTHON36_x86_URL ${prefix}\assets\python36-x86.exe $PYTHON36_x86_SHA256
149 Secure-Download $PYTHON36_x64_URL ${prefix}\assets\python36-x64.exe $PYTHON36_x64_SHA256
149 Secure-Download $PYTHON36_x64_URL ${prefix}\assets\python36-x64.exe $PYTHON36_x64_SHA256
150 Secure-Download $PYTHON37_x86_URL ${prefix}\assets\python37-x86.exe $PYTHON37_x86_SHA256
150 Secure-Download $PYTHON37_x86_URL ${prefix}\assets\python37-x86.exe $PYTHON37_x86_SHA256
151 Secure-Download $PYTHON37_x64_URL ${prefix}\assets\python37-x64.exe $PYTHON37_x64_SHA256
151 Secure-Download $PYTHON37_x64_URL ${prefix}\assets\python37-x64.exe $PYTHON37_x64_SHA256
152 Secure-Download $PYTHON38_x86_URL ${prefix}\assets\python38-x86.exe $PYTHON38_x86_SHA256
152 Secure-Download $PYTHON38_x86_URL ${prefix}\assets\python38-x86.exe $PYTHON38_x86_SHA256
153 Secure-Download $PYTHON38_x64_URL ${prefix}\assets\python38-x64.exe $PYTHON38_x64_SHA256
153 Secure-Download $PYTHON38_x64_URL ${prefix}\assets\python38-x64.exe $PYTHON38_x64_SHA256
154 Secure-Download $PIP_URL ${pip} $PIP_SHA256
154 Secure-Download $PIP_URL ${pip} $PIP_SHA256
155 Secure-Download $VIRTUALENV_URL ${prefix}\assets\virtualenv.tar.gz $VIRTUALENV_SHA256
155 Secure-Download $VIRTUALENV_URL ${prefix}\assets\virtualenv.tar.gz $VIRTUALENV_SHA256
156 Secure-Download $VS_BUILD_TOOLS_URL ${prefix}\assets\vs_buildtools.exe $VS_BUILD_TOOLS_SHA256
156 Secure-Download $VS_BUILD_TOOLS_URL ${prefix}\assets\vs_buildtools.exe $VS_BUILD_TOOLS_SHA256
157 Secure-Download $INNO_SETUP_URL ${prefix}\assets\InnoSetup.exe $INNO_SETUP_SHA256
157 Secure-Download $INNO_SETUP_URL ${prefix}\assets\InnoSetup.exe $INNO_SETUP_SHA256
158 Secure-Download $MINGW_BIN_URL ${prefix}\assets\mingw-get-bin.zip $MINGW_BIN_SHA256
158 Secure-Download $MINGW_BIN_URL ${prefix}\assets\mingw-get-bin.zip $MINGW_BIN_SHA256
159 Secure-Download $MERCURIAL_WHEEL_URL ${prefix}\assets\${MERCURIAL_WHEEL_FILENAME} $MERCURIAL_WHEEL_SHA256
159 Secure-Download $MERCURIAL_WHEEL_URL ${prefix}\assets\${MERCURIAL_WHEEL_FILENAME} $MERCURIAL_WHEEL_SHA256
160 Secure-Download $RUSTUP_INIT_URL ${prefix}\assets\rustup-init.exe $RUSTUP_INIT_SHA256
160 Secure-Download $RUSTUP_INIT_URL ${prefix}\assets\rustup-init.exe $RUSTUP_INIT_SHA256
161
161
162 Write-Output "installing Python 2.7 32-bit"
162 Write-Output "installing Python 2.7 32-bit"
163 Invoke-Process msiexec.exe "/i ${prefix}\assets\python27-x86.msi /l* ${prefix}\assets\python27-x86.log /q TARGETDIR=${prefix}\python27-x86 ALLUSERS="
163 Invoke-Process msiexec.exe "/i ${prefix}\assets\python27-x86.msi /l* ${prefix}\assets\python27-x86.log /q TARGETDIR=${prefix}\python27-x86 ALLUSERS="
164 Invoke-Process ${prefix}\python27-x86\python.exe ${prefix}\assets\get-pip.py
164 Invoke-Process ${prefix}\python27-x86\python.exe ${prefix}\assets\get-pip.py
165 Invoke-Process ${prefix}\python27-x86\Scripts\pip.exe "install ${prefix}\assets\virtualenv.tar.gz"
165 Invoke-Process ${prefix}\python27-x86\Scripts\pip.exe "install ${prefix}\assets\virtualenv.tar.gz"
166
166
167 Write-Output "installing Python 2.7 64-bit"
167 Write-Output "installing Python 2.7 64-bit"
168 Invoke-Process msiexec.exe "/i ${prefix}\assets\python27-x64.msi /l* ${prefix}\assets\python27-x64.log /q TARGETDIR=${prefix}\python27-x64 ALLUSERS="
168 Invoke-Process msiexec.exe "/i ${prefix}\assets\python27-x64.msi /l* ${prefix}\assets\python27-x64.log /q TARGETDIR=${prefix}\python27-x64 ALLUSERS="
169 Invoke-Process ${prefix}\python27-x64\python.exe ${prefix}\assets\get-pip.py
169 Invoke-Process ${prefix}\python27-x64\python.exe ${prefix}\assets\get-pip.py
170 Invoke-Process ${prefix}\python27-x64\Scripts\pip.exe "install ${prefix}\assets\virtualenv.tar.gz"
170 Invoke-Process ${prefix}\python27-x64\Scripts\pip.exe "install ${prefix}\assets\virtualenv.tar.gz"
171
171
172 Install-Python3 "Python 3.5 32-bit" ${prefix}\assets\python35-x86.exe ${prefix}\python35-x86 ${pip}
172 Install-Python3 "Python 3.5 32-bit" ${prefix}\assets\python35-x86.exe ${prefix}\python35-x86 ${pip}
173 Install-Python3 "Python 3.5 64-bit" ${prefix}\assets\python35-x64.exe ${prefix}\python35-x64 ${pip}
173 Install-Python3 "Python 3.5 64-bit" ${prefix}\assets\python35-x64.exe ${prefix}\python35-x64 ${pip}
174 Install-Python3 "Python 3.6 32-bit" ${prefix}\assets\python36-x86.exe ${prefix}\python36-x86 ${pip}
174 Install-Python3 "Python 3.6 32-bit" ${prefix}\assets\python36-x86.exe ${prefix}\python36-x86 ${pip}
175 Install-Python3 "Python 3.6 64-bit" ${prefix}\assets\python36-x64.exe ${prefix}\python36-x64 ${pip}
175 Install-Python3 "Python 3.6 64-bit" ${prefix}\assets\python36-x64.exe ${prefix}\python36-x64 ${pip}
176 Install-Python3 "Python 3.7 32-bit" ${prefix}\assets\python37-x86.exe ${prefix}\python37-x86 ${pip}
176 Install-Python3 "Python 3.7 32-bit" ${prefix}\assets\python37-x86.exe ${prefix}\python37-x86 ${pip}
177 Install-Python3 "Python 3.7 64-bit" ${prefix}\assets\python37-x64.exe ${prefix}\python37-x64 ${pip}
177 Install-Python3 "Python 3.7 64-bit" ${prefix}\assets\python37-x64.exe ${prefix}\python37-x64 ${pip}
178 Install-Python3 "Python 3.8 32-bit" ${prefix}\assets\python38-x86.exe ${prefix}\python38-x86 ${pip}
178 Install-Python3 "Python 3.8 32-bit" ${prefix}\assets\python38-x86.exe ${prefix}\python38-x86 ${pip}
179 Install-Python3 "Python 3.8 64-bit" ${prefix}\assets\python38-x64.exe ${prefix}\python38-x64 ${pip}
179 Install-Python3 "Python 3.8 64-bit" ${prefix}\assets\python38-x64.exe ${prefix}\python38-x64 ${pip}
180
180
181 Write-Output "installing Visual Studio 2017 Build Tools and SDKs"
181 Write-Output "installing Visual Studio 2017 Build Tools and SDKs"
182 Invoke-Process ${prefix}\assets\vs_buildtools.exe "--quiet --wait --norestart --nocache --channelUri https://aka.ms/vs/15/release/channel --add Microsoft.VisualStudio.Workload.MSBuildTools --add Microsoft.VisualStudio.Component.Windows10SDK.17763 --add Microsoft.VisualStudio.Workload.VCTools --add Microsoft.VisualStudio.Component.Windows10SDK --add Microsoft.VisualStudio.Component.VC.140"
182 Invoke-Process ${prefix}\assets\vs_buildtools.exe "--quiet --wait --norestart --nocache --channelUri https://aka.ms/vs/15/release/channel --add Microsoft.VisualStudio.Workload.MSBuildTools --add Microsoft.VisualStudio.Component.Windows10SDK.17763 --add Microsoft.VisualStudio.Workload.VCTools --add Microsoft.VisualStudio.Component.Windows10SDK --add Microsoft.VisualStudio.Component.VC.140"
183
183
184 Install-Rust ${prefix}
184 Install-Rust ${prefix}
185
185
186 Write-Output "installing Visual C++ 9.0 for Python 2.7"
186 Write-Output "installing Visual C++ 9.0 for Python 2.7"
187 Invoke-Process msiexec.exe "/i ${prefix}\assets\VCForPython27.msi /l* ${prefix}\assets\VCForPython27.log /q"
187 Invoke-Process msiexec.exe "/i ${prefix}\assets\VCForPython27.msi /l* ${prefix}\assets\VCForPython27.log /q"
188
188
189 Write-Output "installing Inno Setup"
189 Write-Output "installing Inno Setup"
190 Invoke-Process ${prefix}\assets\InnoSetup.exe "/SP- /VERYSILENT /SUPPRESSMSGBOXES"
190 Invoke-Process ${prefix}\assets\InnoSetup.exe "/SP- /VERYSILENT /SUPPRESSMSGBOXES"
191
191
192 Write-Output "extracting MinGW base archive"
192 Write-Output "extracting MinGW base archive"
193 Expand-Archive -Path ${prefix}\assets\mingw-get-bin.zip -DestinationPath "${prefix}\MinGW" -Force
193 Expand-Archive -Path ${prefix}\assets\mingw-get-bin.zip -DestinationPath "${prefix}\MinGW" -Force
194
194
195 Write-Output "updating MinGW package catalogs"
195 Write-Output "updating MinGW package catalogs"
196 Invoke-Process ${prefix}\MinGW\bin\mingw-get.exe "update"
196 Invoke-Process ${prefix}\MinGW\bin\mingw-get.exe "update"
197
197
198 Write-Output "installing MinGW packages"
198 Write-Output "installing MinGW packages"
199 Invoke-Process ${prefix}\MinGW\bin\mingw-get.exe "install msys-base msys-coreutils msys-diffutils msys-unzip"
199 Invoke-Process ${prefix}\MinGW\bin\mingw-get.exe "install msys-base msys-coreutils msys-diffutils msys-unzip"
200
200
201 # Construct a virtualenv useful for bootstrapping. It conveniently contains a
201 # Construct a virtualenv useful for bootstrapping. It conveniently contains a
202 # Mercurial install.
202 # Mercurial install.
203 Write-Output "creating bootstrap virtualenv with Mercurial"
203 Write-Output "creating bootstrap virtualenv with Mercurial"
204 Invoke-Process "$prefix\python27-x64\Scripts\virtualenv.exe" "${prefix}\venv-bootstrap"
204 Invoke-Process "$prefix\python27-x64\Scripts\virtualenv.exe" "${prefix}\venv-bootstrap"
205 Invoke-Process "${prefix}\venv-bootstrap\Scripts\pip.exe" "install ${prefix}\assets\${MERCURIAL_WHEEL_FILENAME}"
205 Invoke-Process "${prefix}\venv-bootstrap\Scripts\pip.exe" "install ${prefix}\assets\${MERCURIAL_WHEEL_FILENAME}"
206 }
206 }
207
207
208 function Clone-Mercurial-Repo($prefix, $repo_url, $dest) {
208 function Clone-Mercurial-Repo($prefix, $repo_url, $dest) {
209 Write-Output "cloning $repo_url to $dest"
209 Write-Output "cloning $repo_url to $dest"
210 # TODO Figure out why CA verification isn't working in EC2 and remove
210 # TODO Figure out why CA verification isn't working in EC2 and remove
211 # --insecure.
211 # --insecure.
212 Invoke-Process "${prefix}\venv-bootstrap\Scripts\hg.exe" "clone --insecure $repo_url $dest"
212 Invoke-Process "${prefix}\venv-bootstrap\Scripts\hg.exe" "clone --insecure $repo_url $dest"
213
213
214 # Mark repo as non-publishing by default for convenience.
214 # Mark repo as non-publishing by default for convenience.
215 Add-Content -Path "$dest\.hg\hgrc" -Value "`n[phases]`npublish = false"
215 Add-Content -Path "$dest\.hg\hgrc" -Value "`n[phases]`npublish = false"
216 }
216 }
217
217
218 $prefix = "c:\hgdev"
218 $prefix = "c:\hgdev"
219 Install-Dependencies $prefix
219 Install-Dependencies $prefix
220 Clone-Mercurial-Repo $prefix "https://www.mercurial-scm.org/repo/hg" $prefix\src
220 Clone-Mercurial-Repo $prefix "https://www.mercurial-scm.org/repo/hg" $prefix\src
General Comments 0
You need to be logged in to leave comments. Login now