##// END OF EJS Templates
fastannotate: use `get_unique_pull_path`...
marmoute -
r47708:b6b69644 default
parent child Browse files
Show More
@@ -1,260 +1,263 b''
1 # Copyright 2016-present Facebook. All Rights Reserved.
1 # Copyright 2016-present Facebook. All Rights Reserved.
2 #
2 #
3 # protocol: logic for a server providing fastannotate support
3 # protocol: logic for a server providing fastannotate support
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7 from __future__ import absolute_import
7 from __future__ import absolute_import
8
8
9 import contextlib
9 import contextlib
10 import os
10 import os
11
11
12 from mercurial.i18n import _
12 from mercurial.i18n import _
13 from mercurial.pycompat import open
13 from mercurial.pycompat import open
14 from mercurial import (
14 from mercurial import (
15 error,
15 error,
16 extensions,
16 extensions,
17 hg,
17 hg,
18 pycompat,
18 pycompat,
19 util,
19 util,
20 wireprotov1peer,
20 wireprotov1peer,
21 wireprotov1server,
21 wireprotov1server,
22 )
22 )
23 from mercurial.utils import (
24 urlutil,
25 )
23 from . import context
26 from . import context
24
27
25 # common
28 # common
26
29
27
30
28 def _getmaster(ui):
31 def _getmaster(ui):
29 """get the mainbranch, and enforce it is set"""
32 """get the mainbranch, and enforce it is set"""
30 master = ui.config(b'fastannotate', b'mainbranch')
33 master = ui.config(b'fastannotate', b'mainbranch')
31 if not master:
34 if not master:
32 raise error.Abort(
35 raise error.Abort(
33 _(
36 _(
34 b'fastannotate.mainbranch is required '
37 b'fastannotate.mainbranch is required '
35 b'for both the client and the server'
38 b'for both the client and the server'
36 )
39 )
37 )
40 )
38 return master
41 return master
39
42
40
43
41 # server-side
44 # server-side
42
45
43
46
44 def _capabilities(orig, repo, proto):
47 def _capabilities(orig, repo, proto):
45 result = orig(repo, proto)
48 result = orig(repo, proto)
46 result.append(b'getannotate')
49 result.append(b'getannotate')
47 return result
50 return result
48
51
49
52
50 def _getannotate(repo, proto, path, lastnode):
53 def _getannotate(repo, proto, path, lastnode):
51 # output:
54 # output:
52 # FILE := vfspath + '\0' + str(size) + '\0' + content
55 # FILE := vfspath + '\0' + str(size) + '\0' + content
53 # OUTPUT := '' | FILE + OUTPUT
56 # OUTPUT := '' | FILE + OUTPUT
54 result = b''
57 result = b''
55 buildondemand = repo.ui.configbool(
58 buildondemand = repo.ui.configbool(
56 b'fastannotate', b'serverbuildondemand', True
59 b'fastannotate', b'serverbuildondemand', True
57 )
60 )
58 with context.annotatecontext(repo, path) as actx:
61 with context.annotatecontext(repo, path) as actx:
59 if buildondemand:
62 if buildondemand:
60 # update before responding to the client
63 # update before responding to the client
61 master = _getmaster(repo.ui)
64 master = _getmaster(repo.ui)
62 try:
65 try:
63 if not actx.isuptodate(master):
66 if not actx.isuptodate(master):
64 actx.annotate(master, master)
67 actx.annotate(master, master)
65 except Exception:
68 except Exception:
66 # non-fast-forward move or corrupted. rebuild automically.
69 # non-fast-forward move or corrupted. rebuild automically.
67 actx.rebuild()
70 actx.rebuild()
68 try:
71 try:
69 actx.annotate(master, master)
72 actx.annotate(master, master)
70 except Exception:
73 except Exception:
71 actx.rebuild() # delete files
74 actx.rebuild() # delete files
72 finally:
75 finally:
73 # although the "with" context will also do a close/flush, we
76 # although the "with" context will also do a close/flush, we
74 # need to do it early so we can send the correct respond to
77 # need to do it early so we can send the correct respond to
75 # client.
78 # client.
76 actx.close()
79 actx.close()
77 # send back the full content of revmap and linelog, in the future we
80 # send back the full content of revmap and linelog, in the future we
78 # may want to do some rsync-like fancy updating.
81 # may want to do some rsync-like fancy updating.
79 # the lastnode check is not necessary if the client and the server
82 # the lastnode check is not necessary if the client and the server
80 # agree where the main branch is.
83 # agree where the main branch is.
81 if actx.lastnode != lastnode:
84 if actx.lastnode != lastnode:
82 for p in [actx.revmappath, actx.linelogpath]:
85 for p in [actx.revmappath, actx.linelogpath]:
83 if not os.path.exists(p):
86 if not os.path.exists(p):
84 continue
87 continue
85 with open(p, b'rb') as f:
88 with open(p, b'rb') as f:
86 content = f.read()
89 content = f.read()
87 vfsbaselen = len(repo.vfs.base + b'/')
90 vfsbaselen = len(repo.vfs.base + b'/')
88 relpath = p[vfsbaselen:]
91 relpath = p[vfsbaselen:]
89 result += b'%s\0%d\0%s' % (relpath, len(content), content)
92 result += b'%s\0%d\0%s' % (relpath, len(content), content)
90 return result
93 return result
91
94
92
95
93 def _registerwireprotocommand():
96 def _registerwireprotocommand():
94 if b'getannotate' in wireprotov1server.commands:
97 if b'getannotate' in wireprotov1server.commands:
95 return
98 return
96 wireprotov1server.wireprotocommand(b'getannotate', b'path lastnode')(
99 wireprotov1server.wireprotocommand(b'getannotate', b'path lastnode')(
97 _getannotate
100 _getannotate
98 )
101 )
99
102
100
103
101 def serveruisetup(ui):
104 def serveruisetup(ui):
102 _registerwireprotocommand()
105 _registerwireprotocommand()
103 extensions.wrapfunction(wireprotov1server, b'_capabilities', _capabilities)
106 extensions.wrapfunction(wireprotov1server, b'_capabilities', _capabilities)
104
107
105
108
106 # client-side
109 # client-side
107
110
108
111
109 def _parseresponse(payload):
112 def _parseresponse(payload):
110 result = {}
113 result = {}
111 i = 0
114 i = 0
112 l = len(payload) - 1
115 l = len(payload) - 1
113 state = 0 # 0: vfspath, 1: size
116 state = 0 # 0: vfspath, 1: size
114 vfspath = size = b''
117 vfspath = size = b''
115 while i < l:
118 while i < l:
116 ch = payload[i : i + 1]
119 ch = payload[i : i + 1]
117 if ch == b'\0':
120 if ch == b'\0':
118 if state == 1:
121 if state == 1:
119 result[vfspath] = payload[i + 1 : i + 1 + int(size)]
122 result[vfspath] = payload[i + 1 : i + 1 + int(size)]
120 i += int(size)
123 i += int(size)
121 state = 0
124 state = 0
122 vfspath = size = b''
125 vfspath = size = b''
123 elif state == 0:
126 elif state == 0:
124 state = 1
127 state = 1
125 else:
128 else:
126 if state == 1:
129 if state == 1:
127 size += ch
130 size += ch
128 elif state == 0:
131 elif state == 0:
129 vfspath += ch
132 vfspath += ch
130 i += 1
133 i += 1
131 return result
134 return result
132
135
133
136
134 def peersetup(ui, peer):
137 def peersetup(ui, peer):
135 class fastannotatepeer(peer.__class__):
138 class fastannotatepeer(peer.__class__):
136 @wireprotov1peer.batchable
139 @wireprotov1peer.batchable
137 def getannotate(self, path, lastnode=None):
140 def getannotate(self, path, lastnode=None):
138 if not self.capable(b'getannotate'):
141 if not self.capable(b'getannotate'):
139 ui.warn(_(b'remote peer cannot provide annotate cache\n'))
142 ui.warn(_(b'remote peer cannot provide annotate cache\n'))
140 yield None, None
143 yield None, None
141 else:
144 else:
142 args = {b'path': path, b'lastnode': lastnode or b''}
145 args = {b'path': path, b'lastnode': lastnode or b''}
143 f = wireprotov1peer.future()
146 f = wireprotov1peer.future()
144 yield args, f
147 yield args, f
145 yield _parseresponse(f.value)
148 yield _parseresponse(f.value)
146
149
147 peer.__class__ = fastannotatepeer
150 peer.__class__ = fastannotatepeer
148
151
149
152
150 @contextlib.contextmanager
153 @contextlib.contextmanager
151 def annotatepeer(repo):
154 def annotatepeer(repo):
152 ui = repo.ui
155 ui = repo.ui
153
156
154 remotepath = ui.expandpath(
157 remotedest = ui.config(b'fastannotate', b'remotepath', b'default')
155 ui.config(b'fastannotate', b'remotepath', b'default')
158 r = urlutil.get_unique_pull_path(b'fastannotate', repo, ui, remotedest)
156 )
159 remotepath = r[0]
157 peer = hg.peer(ui, {}, remotepath)
160 peer = hg.peer(ui, {}, remotepath)
158
161
159 try:
162 try:
160 yield peer
163 yield peer
161 finally:
164 finally:
162 peer.close()
165 peer.close()
163
166
164
167
165 def clientfetch(repo, paths, lastnodemap=None, peer=None):
168 def clientfetch(repo, paths, lastnodemap=None, peer=None):
166 """download annotate cache from the server for paths"""
169 """download annotate cache from the server for paths"""
167 if not paths:
170 if not paths:
168 return
171 return
169
172
170 if peer is None:
173 if peer is None:
171 with annotatepeer(repo) as peer:
174 with annotatepeer(repo) as peer:
172 return clientfetch(repo, paths, lastnodemap, peer)
175 return clientfetch(repo, paths, lastnodemap, peer)
173
176
174 if lastnodemap is None:
177 if lastnodemap is None:
175 lastnodemap = {}
178 lastnodemap = {}
176
179
177 ui = repo.ui
180 ui = repo.ui
178 results = []
181 results = []
179 with peer.commandexecutor() as batcher:
182 with peer.commandexecutor() as batcher:
180 ui.debug(b'fastannotate: requesting %d files\n' % len(paths))
183 ui.debug(b'fastannotate: requesting %d files\n' % len(paths))
181 for p in paths:
184 for p in paths:
182 results.append(
185 results.append(
183 batcher.callcommand(
186 batcher.callcommand(
184 b'getannotate',
187 b'getannotate',
185 {b'path': p, b'lastnode': lastnodemap.get(p)},
188 {b'path': p, b'lastnode': lastnodemap.get(p)},
186 )
189 )
187 )
190 )
188
191
189 for result in results:
192 for result in results:
190 r = result.result()
193 r = result.result()
191 # TODO: pconvert these paths on the server?
194 # TODO: pconvert these paths on the server?
192 r = {util.pconvert(p): v for p, v in pycompat.iteritems(r)}
195 r = {util.pconvert(p): v for p, v in pycompat.iteritems(r)}
193 for path in sorted(r):
196 for path in sorted(r):
194 # ignore malicious paths
197 # ignore malicious paths
195 if not path.startswith(b'fastannotate/') or b'/../' in (
198 if not path.startswith(b'fastannotate/') or b'/../' in (
196 path + b'/'
199 path + b'/'
197 ):
200 ):
198 ui.debug(
201 ui.debug(
199 b'fastannotate: ignored malicious path %s\n' % path
202 b'fastannotate: ignored malicious path %s\n' % path
200 )
203 )
201 continue
204 continue
202 content = r[path]
205 content = r[path]
203 if ui.debugflag:
206 if ui.debugflag:
204 ui.debug(
207 ui.debug(
205 b'fastannotate: writing %d bytes to %s\n'
208 b'fastannotate: writing %d bytes to %s\n'
206 % (len(content), path)
209 % (len(content), path)
207 )
210 )
208 repo.vfs.makedirs(os.path.dirname(path))
211 repo.vfs.makedirs(os.path.dirname(path))
209 with repo.vfs(path, b'wb') as f:
212 with repo.vfs(path, b'wb') as f:
210 f.write(content)
213 f.write(content)
211
214
212
215
213 def _filterfetchpaths(repo, paths):
216 def _filterfetchpaths(repo, paths):
214 """return a subset of paths whose history is long and need to fetch linelog
217 """return a subset of paths whose history is long and need to fetch linelog
215 from the server. works with remotefilelog and non-remotefilelog repos.
218 from the server. works with remotefilelog and non-remotefilelog repos.
216 """
219 """
217 threshold = repo.ui.configint(b'fastannotate', b'clientfetchthreshold', 10)
220 threshold = repo.ui.configint(b'fastannotate', b'clientfetchthreshold', 10)
218 if threshold <= 0:
221 if threshold <= 0:
219 return paths
222 return paths
220
223
221 result = []
224 result = []
222 for path in paths:
225 for path in paths:
223 try:
226 try:
224 if len(repo.file(path)) >= threshold:
227 if len(repo.file(path)) >= threshold:
225 result.append(path)
228 result.append(path)
226 except Exception: # file not found etc.
229 except Exception: # file not found etc.
227 result.append(path)
230 result.append(path)
228
231
229 return result
232 return result
230
233
231
234
232 def localreposetup(ui, repo):
235 def localreposetup(ui, repo):
233 class fastannotaterepo(repo.__class__):
236 class fastannotaterepo(repo.__class__):
234 def prefetchfastannotate(self, paths, peer=None):
237 def prefetchfastannotate(self, paths, peer=None):
235 master = _getmaster(self.ui)
238 master = _getmaster(self.ui)
236 needupdatepaths = []
239 needupdatepaths = []
237 lastnodemap = {}
240 lastnodemap = {}
238 try:
241 try:
239 for path in _filterfetchpaths(self, paths):
242 for path in _filterfetchpaths(self, paths):
240 with context.annotatecontext(self, path) as actx:
243 with context.annotatecontext(self, path) as actx:
241 if not actx.isuptodate(master, strict=False):
244 if not actx.isuptodate(master, strict=False):
242 needupdatepaths.append(path)
245 needupdatepaths.append(path)
243 lastnodemap[path] = actx.lastnode
246 lastnodemap[path] = actx.lastnode
244 if needupdatepaths:
247 if needupdatepaths:
245 clientfetch(self, needupdatepaths, lastnodemap, peer)
248 clientfetch(self, needupdatepaths, lastnodemap, peer)
246 except Exception as ex:
249 except Exception as ex:
247 # could be directory not writable or so, not fatal
250 # could be directory not writable or so, not fatal
248 self.ui.debug(b'fastannotate: prefetch failed: %r\n' % ex)
251 self.ui.debug(b'fastannotate: prefetch failed: %r\n' % ex)
249
252
250 repo.__class__ = fastannotaterepo
253 repo.__class__ = fastannotaterepo
251
254
252
255
253 def clientreposetup(ui, repo):
256 def clientreposetup(ui, repo):
254 _registerwireprotocommand()
257 _registerwireprotocommand()
255 if repo.local():
258 if repo.local():
256 localreposetup(ui, repo)
259 localreposetup(ui, repo)
257 # TODO: this mutates global state, but only if at least one repo
260 # TODO: this mutates global state, but only if at least one repo
258 # has the extension enabled. This is probably bad for hgweb.
261 # has the extension enabled. This is probably bad for hgweb.
259 if peersetup not in hg.wirepeersetupfuncs:
262 if peersetup not in hg.wirepeersetupfuncs:
260 hg.wirepeersetupfuncs.append(peersetup)
263 hg.wirepeersetupfuncs.append(peersetup)
General Comments 0
You need to be logged in to leave comments. Login now