##// END OF EJS Templates
Allow checking for backports via milestone
Skipper Seabold -
Show More
@@ -1,143 +1,159 b''
1 1 #!/usr/bin/env python
2 2 """
3 3 Backport pull requests to a particular branch.
4 4
5 5 Usage: backport_pr.py branch [PR]
6 6
7 7 e.g.:
8 8
9 9 python tools/backport_pr.py 0.13.1 123
10 10
11 11 to backport PR #123 onto branch 0.13.1
12 12
13 13 or
14 14
15 15 python tools/backport_pr.py 1.x
16 16
17 17 to see what PRs are marked for backport that have yet to be applied.
18 18
19 19 """
20 20
21 21 from __future__ import print_function
22 22
23 23 import os
24 24 import re
25 25 import sys
26 26
27 27 from subprocess import Popen, PIPE, check_call, check_output
28 28 from urllib import urlopen
29 29
30 30 from gh_api import (
31 31 get_issues_list,
32 32 get_pull_request,
33 33 get_pull_request_files,
34 34 is_pull_request,
35 get_milestone_id,
35 36 )
36 37
37 38 def find_rejects(root='.'):
38 39 for dirname, dirs, files in os.walk(root):
39 40 for fname in files:
40 41 if fname.endswith('.rej'):
41 42 yield os.path.join(dirname, fname)
42 43
43 44 def get_current_branch():
44 45 branches = check_output(['git', 'branch'])
45 46 for branch in branches.splitlines():
46 47 if branch.startswith('*'):
47 48 return branch[1:].strip()
48 49
49 50 def backport_pr(branch, num, project='ipython/ipython'):
50 51 current_branch = get_current_branch()
51 52 if branch != current_branch:
52 53 check_call(['git', 'checkout', branch])
53 54 check_call(['git', 'pull'])
54 55 pr = get_pull_request(project, num, auth=True)
55 56 files = get_pull_request_files(project, num, auth=True)
56 57 patch_url = pr['patch_url']
57 58 title = pr['title']
58 59 description = pr['body']
59 60 fname = "PR%i.patch" % num
60 61 if os.path.exists(fname):
61 62 print("using patch from {fname}".format(**locals()))
62 63 with open(fname) as f:
63 64 patch = f.read()
64 65 else:
65 66 req = urlopen(patch_url)
66 67 patch = req.read()
67 68
68 69 msg = "Backport PR #%i: %s" % (num, title) + '\n\n' + description
69 70 check = Popen(['git', 'apply', '--check', '--verbose'], stdin=PIPE)
70 71 a,b = check.communicate(patch)
71 72
72 73 if check.returncode:
73 74 print("patch did not apply, saving to {fname}".format(**locals()))
74 75 print("edit {fname} until `cat {fname} | git apply --check` succeeds".format(**locals()))
75 76 print("then run tools/backport_pr.py {num} again".format(**locals()))
76 77 if not os.path.exists(fname):
77 78 with open(fname, 'wb') as f:
78 79 f.write(patch)
79 80 return 1
80 81
81 82 p = Popen(['git', 'apply'], stdin=PIPE)
82 83 a,b = p.communicate(patch)
83 84
84 85 filenames = [ f['filename'] for f in files ]
85 86
86 87 check_call(['git', 'add'] + filenames)
87 88
88 89 check_call(['git', 'commit', '-m', msg])
89 90
90 91 print("PR #%i applied, with msg:" % num)
91 92 print()
92 93 print(msg)
93 94 print()
94 95
95 96 if branch != current_branch:
96 97 check_call(['git', 'checkout', current_branch])
97 98
98 99 return 0
99 100
100 101 backport_re = re.compile(r"[Bb]ackport.*?(\d+)")
101 102
102 103 def already_backported(branch, since_tag=None):
103 104 """return set of PRs that have been backported already"""
104 105 if since_tag is None:
105 106 since_tag = check_output(['git','describe', branch, '--abbrev=0']).decode('utf8').strip()
106 107 cmd = ['git', 'log', '%s..%s' % (since_tag, branch), '--oneline']
107 108 lines = check_output(cmd).decode('utf8')
108 109 return set(int(num) for num in backport_re.findall(lines))
109 110
110 def should_backport(labels):
111 def should_backport(labels=None, milestone=None):
111 112 """return set of PRs marked for backport"""
113 if labels is None and milestone is None:
114 raise ValueError("Specify one of labels or milestone.")
115 elif labels is not None and milestone is not None:
116 raise ValueError("Specify only one of labels or milestone.")
117 if labels is not None:
112 118 issues = get_issues_list("ipython/ipython",
113 119 labels=labels,
114 120 state='closed',
115 121 auth=True,
116 122 )
123 else:
124 milestone_id = get_milestone_id("ipython/ipython", milestone,
125 auth=True)
126 issues = get_issues_list("ipython/ipython",
127 milestone=milestone_id,
128 state='closed',
129 auth=True,
130 )
131
117 132 should_backport = set()
118 133 for issue in issues:
119 134 if not is_pull_request(issue):
120 135 continue
121 pr = get_pull_request("ipython/ipython", issue['number'], auth=True)
136 pr = get_pull_request("ipython/ipython", issue['number'],
137 auth=True)
122 138 if not pr['merged']:
123 139 print ("Marked PR closed without merge: %i" % pr['number'])
124 140 continue
125 141 should_backport.add(pr['number'])
126 142 return should_backport
127 143
128 144 if __name__ == '__main__':
129 145
130 146 if len(sys.argv) < 2:
131 147 print(__doc__)
132 148 sys.exit(1)
133 149
134 150 if len(sys.argv) < 3:
135 151 branch = sys.argv[1]
136 152 already = already_backported(branch)
137 153 should = should_backport("backport-1.2")
138 154 print ("The following PRs should be backported:")
139 155 for pr in should.difference(already):
140 156 print (pr)
141 157 sys.exit(0)
142 158
143 159 sys.exit(backport_pr(sys.argv[1], int(sys.argv[2])))
@@ -1,259 +1,276 b''
1 1 """Functions for Github API requests."""
2 2 from __future__ import print_function
3 3
4 4 try:
5 5 input = raw_input
6 6 except NameError:
7 7 pass
8 8
9 9 import os
10 10 import re
11 11 import sys
12 12
13 13 import requests
14 14 import getpass
15 15 import json
16 16
17 17 try:
18 18 import requests_cache
19 19 except ImportError:
20 20 print("no cache")
21 21 else:
22 22 requests_cache.install_cache("gh_api")
23 23
24 24 # Keyring stores passwords by a 'username', but we're not storing a username and
25 25 # password
26 26 fake_username = 'ipython_tools'
27 27
28 28 class Obj(dict):
29 29 """Dictionary with attribute access to names."""
30 30 def __getattr__(self, name):
31 31 try:
32 32 return self[name]
33 33 except KeyError:
34 34 raise AttributeError(name)
35 35
36 36 def __setattr__(self, name, val):
37 37 self[name] = val
38 38
39 39 token = None
40 40 def get_auth_token():
41 41 global token
42 42
43 43 if token is not None:
44 44 return token
45 45
46 46 import keyring
47 47 token = keyring.get_password('github', fake_username)
48 48 if token is not None:
49 49 return token
50 50
51 51 print("Please enter your github username and password. These are not "
52 52 "stored, only used to get an oAuth token. You can revoke this at "
53 53 "any time on Github.")
54 54 user = input("Username: ")
55 55 pw = getpass.getpass("Password: ")
56 56
57 57 auth_request = {
58 58 "scopes": [
59 59 "public_repo",
60 60 "gist"
61 61 ],
62 62 "note": "IPython tools",
63 63 "note_url": "https://github.com/ipython/ipython/tree/master/tools",
64 64 }
65 65 response = requests.post('https://api.github.com/authorizations',
66 66 auth=(user, pw), data=json.dumps(auth_request))
67 67 response.raise_for_status()
68 68 token = json.loads(response.text)['token']
69 69 keyring.set_password('github', fake_username, token)
70 70 return token
71 71
72 72 def make_auth_header():
73 73 return {'Authorization': 'token ' + get_auth_token()}
74 74
75 75 def post_issue_comment(project, num, body):
76 76 url = 'https://api.github.com/repos/{project}/issues/{num}/comments'.format(project=project, num=num)
77 77 payload = json.dumps({'body': body})
78 78 requests.post(url, data=payload, headers=make_auth_header())
79 79
80 80 def post_gist(content, description='', filename='file', auth=False):
81 81 """Post some text to a Gist, and return the URL."""
82 82 post_data = json.dumps({
83 83 "description": description,
84 84 "public": True,
85 85 "files": {
86 86 filename: {
87 87 "content": content
88 88 }
89 89 }
90 90 }).encode('utf-8')
91 91
92 92 headers = make_auth_header() if auth else {}
93 93 response = requests.post("https://api.github.com/gists", data=post_data, headers=headers)
94 94 response.raise_for_status()
95 95 response_data = json.loads(response.text)
96 96 return response_data['html_url']
97 97
98 98 def get_pull_request(project, num, auth=False):
99 99 """get pull request info by number
100 100 """
101 101 url = "https://api.github.com/repos/{project}/pulls/{num}".format(project=project, num=num)
102 102 if auth:
103 103 header = make_auth_header()
104 104 else:
105 105 header = None
106 106 response = requests.get(url, headers=header)
107 107 response.raise_for_status()
108 108 return json.loads(response.text, object_hook=Obj)
109 109
110 110 def get_pull_request_files(project, num, auth=False):
111 111 """get list of files in a pull request"""
112 112 url = "https://api.github.com/repos/{project}/pulls/{num}/files".format(project=project, num=num)
113 113 if auth:
114 114 header = make_auth_header()
115 115 else:
116 116 header = None
117 117 return get_paged_request(url, headers=header)
118 118
119 119 element_pat = re.compile(r'<(.+?)>')
120 120 rel_pat = re.compile(r'rel=[\'"](\w+)[\'"]')
121 121
122 122 def get_paged_request(url, headers=None, **params):
123 123 """get a full list, handling APIv3's paging"""
124 124 results = []
125 125 params.setdefault("per_page", 100)
126 126 while True:
127 127 print("fetching %s with %s" % (url, params), file=sys.stderr)
128 128 response = requests.get(url, headers=headers, params=params)
129 129 response.raise_for_status()
130 130 results.extend(response.json())
131 131 if 'next' in response.links:
132 132 url = response.links['next']['url']
133 133 else:
134 134 break
135 135 return results
136 136
137 137 def get_pulls_list(project, auth=False, **params):
138 138 """get pull request list"""
139 139 params.setdefault("state", "closed")
140 140 url = "https://api.github.com/repos/{project}/pulls".format(project=project)
141 141 if auth:
142 142 headers = make_auth_header()
143 143 else:
144 144 headers = None
145 145 pages = get_paged_request(url, headers=headers, params=params)
146 146 return pages
147 147
148 148 def get_issues_list(project, auth=False, **params):
149 149 """get issues list"""
150 150 params.setdefault("state", "closed")
151 151 url = "https://api.github.com/repos/{project}/issues".format(project=project)
152 152 if auth:
153 153 headers = make_auth_header()
154 154 else:
155 155 headers = None
156 156 pages = get_paged_request(url, headers=headers, **params)
157 157 return pages
158 158
159 def get_milestones(project, auth=False, **params):
160 url = "https://api.github.com/repos/{project}/milestones".format(project=project)
161 if auth:
162 headers = make_auth_header()
163 else:
164 headers = None
165 pages = get_paged_request(url, headers=headers, **params)
166 return pages
167
168 def get_milestone_id(project, milestone, auth=False, **params):
169 pages = get_milestones(project, auth=auth, **params)
170 for page in pages:
171 if page['title'] == milestone:
172 return page['number']
173 else:
174 raise ValueError("milestone %s not found" % milestone)
175
159 176 def is_pull_request(issue):
160 177 """Return True if the given issue is a pull request."""
161 178 return bool(issue.get('pull_request', {}).get('html_url', None))
162 179
163 180 # encode_multipart_formdata is from urllib3.filepost
164 181 # The only change is to iter_fields, to enforce S3's required key ordering
165 182
166 183 def iter_fields(fields):
167 184 fields = fields.copy()
168 185 for key in ('key', 'acl', 'Filename', 'success_action_status', 'AWSAccessKeyId',
169 186 'Policy', 'Signature', 'Content-Type', 'file'):
170 187 yield (key, fields.pop(key))
171 188 for (k,v) in fields.items():
172 189 yield k,v
173 190
174 191 def encode_multipart_formdata(fields, boundary=None):
175 192 """
176 193 Encode a dictionary of ``fields`` using the multipart/form-data mime format.
177 194
178 195 :param fields:
179 196 Dictionary of fields or list of (key, value) field tuples. The key is
180 197 treated as the field name, and the value as the body of the form-data
181 198 bytes. If the value is a tuple of two elements, then the first element
182 199 is treated as the filename of the form-data section.
183 200
184 201 Field names and filenames must be unicode.
185 202
186 203 :param boundary:
187 204 If not specified, then a random boundary will be generated using
188 205 :func:`mimetools.choose_boundary`.
189 206 """
190 207 # copy requests imports in here:
191 208 from io import BytesIO
192 209 from requests.packages.urllib3.filepost import (
193 210 choose_boundary, six, writer, b, get_content_type
194 211 )
195 212 body = BytesIO()
196 213 if boundary is None:
197 214 boundary = choose_boundary()
198 215
199 216 for fieldname, value in iter_fields(fields):
200 217 body.write(b('--%s\r\n' % (boundary)))
201 218
202 219 if isinstance(value, tuple):
203 220 filename, data = value
204 221 writer(body).write('Content-Disposition: form-data; name="%s"; '
205 222 'filename="%s"\r\n' % (fieldname, filename))
206 223 body.write(b('Content-Type: %s\r\n\r\n' %
207 224 (get_content_type(filename))))
208 225 else:
209 226 data = value
210 227 writer(body).write('Content-Disposition: form-data; name="%s"\r\n'
211 228 % (fieldname))
212 229 body.write(b'Content-Type: text/plain\r\n\r\n')
213 230
214 231 if isinstance(data, int):
215 232 data = str(data) # Backwards compatibility
216 233 if isinstance(data, six.text_type):
217 234 writer(body).write(data)
218 235 else:
219 236 body.write(data)
220 237
221 238 body.write(b'\r\n')
222 239
223 240 body.write(b('--%s--\r\n' % (boundary)))
224 241
225 242 content_type = b('multipart/form-data; boundary=%s' % boundary)
226 243
227 244 return body.getvalue(), content_type
228 245
229 246
230 247 def post_download(project, filename, name=None, description=""):
231 248 """Upload a file to the GitHub downloads area"""
232 249 if name is None:
233 250 name = os.path.basename(filename)
234 251 with open(filename, 'rb') as f:
235 252 filedata = f.read()
236 253
237 254 url = "https://api.github.com/repos/{project}/downloads".format(project=project)
238 255
239 256 payload = json.dumps(dict(name=name, size=len(filedata),
240 257 description=description))
241 258 response = requests.post(url, data=payload, headers=make_auth_header())
242 259 response.raise_for_status()
243 260 reply = json.loads(response.content)
244 261 s3_url = reply['s3_url']
245 262
246 263 fields = dict(
247 264 key=reply['path'],
248 265 acl=reply['acl'],
249 266 success_action_status=201,
250 267 Filename=reply['name'],
251 268 AWSAccessKeyId=reply['accesskeyid'],
252 269 Policy=reply['policy'],
253 270 Signature=reply['signature'],
254 271 file=(reply['name'], filedata),
255 272 )
256 273 fields['Content-Type'] = reply['mime_type']
257 274 data, content_type = encode_multipart_formdata(fields)
258 275 s3r = requests.post(s3_url, data=data, headers={'Content-Type': content_type})
259 276 return s3r
General Comments 0
You need to be logged in to leave comments. Login now