##// END OF EJS Templates
path: pass `path` to `peer` in `hg fastannotate`...
marmoute -
r50630:8c6f7839 default
parent child Browse files
Show More
@@ -1,259 +1,262 b''
1 # Copyright 2016-present Facebook. All Rights Reserved.
1 # Copyright 2016-present Facebook. All Rights Reserved.
2 #
2 #
3 # protocol: logic for a server providing fastannotate support
3 # protocol: logic for a server providing fastannotate support
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 import contextlib
8 import contextlib
9 import os
9 import os
10
10
11 from mercurial.i18n import _
11 from mercurial.i18n import _
12 from mercurial.pycompat import open
12 from mercurial.pycompat import open
13 from mercurial import (
13 from mercurial import (
14 error,
14 error,
15 extensions,
15 extensions,
16 hg,
16 hg,
17 util,
17 util,
18 wireprotov1peer,
18 wireprotov1peer,
19 wireprotov1server,
19 wireprotov1server,
20 )
20 )
21 from mercurial.utils import (
21 from mercurial.utils import (
22 urlutil,
22 urlutil,
23 )
23 )
24 from . import context
24 from . import context
25
25
26 # common
26 # common
27
27
28
28
29 def _getmaster(ui):
29 def _getmaster(ui):
30 """get the mainbranch, and enforce it is set"""
30 """get the mainbranch, and enforce it is set"""
31 master = ui.config(b'fastannotate', b'mainbranch')
31 master = ui.config(b'fastannotate', b'mainbranch')
32 if not master:
32 if not master:
33 raise error.Abort(
33 raise error.Abort(
34 _(
34 _(
35 b'fastannotate.mainbranch is required '
35 b'fastannotate.mainbranch is required '
36 b'for both the client and the server'
36 b'for both the client and the server'
37 )
37 )
38 )
38 )
39 return master
39 return master
40
40
41
41
42 # server-side
42 # server-side
43
43
44
44
45 def _capabilities(orig, repo, proto):
45 def _capabilities(orig, repo, proto):
46 result = orig(repo, proto)
46 result = orig(repo, proto)
47 result.append(b'getannotate')
47 result.append(b'getannotate')
48 return result
48 return result
49
49
50
50
51 def _getannotate(repo, proto, path, lastnode):
51 def _getannotate(repo, proto, path, lastnode):
52 # output:
52 # output:
53 # FILE := vfspath + '\0' + str(size) + '\0' + content
53 # FILE := vfspath + '\0' + str(size) + '\0' + content
54 # OUTPUT := '' | FILE + OUTPUT
54 # OUTPUT := '' | FILE + OUTPUT
55 result = b''
55 result = b''
56 buildondemand = repo.ui.configbool(
56 buildondemand = repo.ui.configbool(
57 b'fastannotate', b'serverbuildondemand', True
57 b'fastannotate', b'serverbuildondemand', True
58 )
58 )
59 with context.annotatecontext(repo, path) as actx:
59 with context.annotatecontext(repo, path) as actx:
60 if buildondemand:
60 if buildondemand:
61 # update before responding to the client
61 # update before responding to the client
62 master = _getmaster(repo.ui)
62 master = _getmaster(repo.ui)
63 try:
63 try:
64 if not actx.isuptodate(master):
64 if not actx.isuptodate(master):
65 actx.annotate(master, master)
65 actx.annotate(master, master)
66 except Exception:
66 except Exception:
67 # non-fast-forward move or corrupted. rebuild automically.
67 # non-fast-forward move or corrupted. rebuild automically.
68 actx.rebuild()
68 actx.rebuild()
69 try:
69 try:
70 actx.annotate(master, master)
70 actx.annotate(master, master)
71 except Exception:
71 except Exception:
72 actx.rebuild() # delete files
72 actx.rebuild() # delete files
73 finally:
73 finally:
74 # although the "with" context will also do a close/flush, we
74 # although the "with" context will also do a close/flush, we
75 # need to do it early so we can send the correct respond to
75 # need to do it early so we can send the correct respond to
76 # client.
76 # client.
77 actx.close()
77 actx.close()
78 # send back the full content of revmap and linelog, in the future we
78 # send back the full content of revmap and linelog, in the future we
79 # may want to do some rsync-like fancy updating.
79 # may want to do some rsync-like fancy updating.
80 # the lastnode check is not necessary if the client and the server
80 # the lastnode check is not necessary if the client and the server
81 # agree where the main branch is.
81 # agree where the main branch is.
82 if actx.lastnode != lastnode:
82 if actx.lastnode != lastnode:
83 for p in [actx.revmappath, actx.linelogpath]:
83 for p in [actx.revmappath, actx.linelogpath]:
84 if not os.path.exists(p):
84 if not os.path.exists(p):
85 continue
85 continue
86 with open(p, b'rb') as f:
86 with open(p, b'rb') as f:
87 content = f.read()
87 content = f.read()
88 vfsbaselen = len(repo.vfs.base + b'/')
88 vfsbaselen = len(repo.vfs.base + b'/')
89 relpath = p[vfsbaselen:]
89 relpath = p[vfsbaselen:]
90 result += b'%s\0%d\0%s' % (relpath, len(content), content)
90 result += b'%s\0%d\0%s' % (relpath, len(content), content)
91 return result
91 return result
92
92
93
93
94 def _registerwireprotocommand():
94 def _registerwireprotocommand():
95 if b'getannotate' in wireprotov1server.commands:
95 if b'getannotate' in wireprotov1server.commands:
96 return
96 return
97 wireprotov1server.wireprotocommand(b'getannotate', b'path lastnode')(
97 wireprotov1server.wireprotocommand(b'getannotate', b'path lastnode')(
98 _getannotate
98 _getannotate
99 )
99 )
100
100
101
101
102 def serveruisetup(ui):
102 def serveruisetup(ui):
103 _registerwireprotocommand()
103 _registerwireprotocommand()
104 extensions.wrapfunction(wireprotov1server, b'_capabilities', _capabilities)
104 extensions.wrapfunction(wireprotov1server, b'_capabilities', _capabilities)
105
105
106
106
107 # client-side
107 # client-side
108
108
109
109
110 def _parseresponse(payload):
110 def _parseresponse(payload):
111 result = {}
111 result = {}
112 i = 0
112 i = 0
113 l = len(payload) - 1
113 l = len(payload) - 1
114 state = 0 # 0: vfspath, 1: size
114 state = 0 # 0: vfspath, 1: size
115 vfspath = size = b''
115 vfspath = size = b''
116 while i < l:
116 while i < l:
117 ch = payload[i : i + 1]
117 ch = payload[i : i + 1]
118 if ch == b'\0':
118 if ch == b'\0':
119 if state == 1:
119 if state == 1:
120 result[vfspath] = payload[i + 1 : i + 1 + int(size)]
120 result[vfspath] = payload[i + 1 : i + 1 + int(size)]
121 i += int(size)
121 i += int(size)
122 state = 0
122 state = 0
123 vfspath = size = b''
123 vfspath = size = b''
124 elif state == 0:
124 elif state == 0:
125 state = 1
125 state = 1
126 else:
126 else:
127 if state == 1:
127 if state == 1:
128 size += ch
128 size += ch
129 elif state == 0:
129 elif state == 0:
130 vfspath += ch
130 vfspath += ch
131 i += 1
131 i += 1
132 return result
132 return result
133
133
134
134
135 def peersetup(ui, peer):
135 def peersetup(ui, peer):
136 class fastannotatepeer(peer.__class__):
136 class fastannotatepeer(peer.__class__):
137 @wireprotov1peer.batchable
137 @wireprotov1peer.batchable
138 def getannotate(self, path, lastnode=None):
138 def getannotate(self, path, lastnode=None):
139 if not self.capable(b'getannotate'):
139 if not self.capable(b'getannotate'):
140 ui.warn(_(b'remote peer cannot provide annotate cache\n'))
140 ui.warn(_(b'remote peer cannot provide annotate cache\n'))
141 return None, None
141 return None, None
142 else:
142 else:
143 args = {b'path': path, b'lastnode': lastnode or b''}
143 args = {b'path': path, b'lastnode': lastnode or b''}
144 return args, _parseresponse
144 return args, _parseresponse
145
145
146 peer.__class__ = fastannotatepeer
146 peer.__class__ = fastannotatepeer
147
147
148
148
149 @contextlib.contextmanager
149 @contextlib.contextmanager
150 def annotatepeer(repo):
150 def annotatepeer(repo):
151 ui = repo.ui
151 ui = repo.ui
152
152
153 remotedest = ui.config(b'fastannotate', b'remotepath', b'default')
153 remotedest = ui.config(b'fastannotate', b'remotepath', b'default')
154 r = urlutil.get_unique_pull_path(b'fastannotate', repo, ui, remotedest)
154 remotepath = urlutil.get_unique_pull_path_obj(
155 remotepath = r[0]
155 b'fastannotate',
156 ui,
157 remotedest,
158 )
156 peer = hg.peer(ui, {}, remotepath)
159 peer = hg.peer(ui, {}, remotepath)
157
160
158 try:
161 try:
159 yield peer
162 yield peer
160 finally:
163 finally:
161 peer.close()
164 peer.close()
162
165
163
166
164 def clientfetch(repo, paths, lastnodemap=None, peer=None):
167 def clientfetch(repo, paths, lastnodemap=None, peer=None):
165 """download annotate cache from the server for paths"""
168 """download annotate cache from the server for paths"""
166 if not paths:
169 if not paths:
167 return
170 return
168
171
169 if peer is None:
172 if peer is None:
170 with annotatepeer(repo) as peer:
173 with annotatepeer(repo) as peer:
171 return clientfetch(repo, paths, lastnodemap, peer)
174 return clientfetch(repo, paths, lastnodemap, peer)
172
175
173 if lastnodemap is None:
176 if lastnodemap is None:
174 lastnodemap = {}
177 lastnodemap = {}
175
178
176 ui = repo.ui
179 ui = repo.ui
177 results = []
180 results = []
178 with peer.commandexecutor() as batcher:
181 with peer.commandexecutor() as batcher:
179 ui.debug(b'fastannotate: requesting %d files\n' % len(paths))
182 ui.debug(b'fastannotate: requesting %d files\n' % len(paths))
180 for p in paths:
183 for p in paths:
181 results.append(
184 results.append(
182 batcher.callcommand(
185 batcher.callcommand(
183 b'getannotate',
186 b'getannotate',
184 {b'path': p, b'lastnode': lastnodemap.get(p)},
187 {b'path': p, b'lastnode': lastnodemap.get(p)},
185 )
188 )
186 )
189 )
187
190
188 for result in results:
191 for result in results:
189 r = result.result()
192 r = result.result()
190 # TODO: pconvert these paths on the server?
193 # TODO: pconvert these paths on the server?
191 r = {util.pconvert(p): v for p, v in r.items()}
194 r = {util.pconvert(p): v for p, v in r.items()}
192 for path in sorted(r):
195 for path in sorted(r):
193 # ignore malicious paths
196 # ignore malicious paths
194 if not path.startswith(b'fastannotate/') or b'/../' in (
197 if not path.startswith(b'fastannotate/') or b'/../' in (
195 path + b'/'
198 path + b'/'
196 ):
199 ):
197 ui.debug(
200 ui.debug(
198 b'fastannotate: ignored malicious path %s\n' % path
201 b'fastannotate: ignored malicious path %s\n' % path
199 )
202 )
200 continue
203 continue
201 content = r[path]
204 content = r[path]
202 if ui.debugflag:
205 if ui.debugflag:
203 ui.debug(
206 ui.debug(
204 b'fastannotate: writing %d bytes to %s\n'
207 b'fastannotate: writing %d bytes to %s\n'
205 % (len(content), path)
208 % (len(content), path)
206 )
209 )
207 repo.vfs.makedirs(os.path.dirname(path))
210 repo.vfs.makedirs(os.path.dirname(path))
208 with repo.vfs(path, b'wb') as f:
211 with repo.vfs(path, b'wb') as f:
209 f.write(content)
212 f.write(content)
210
213
211
214
212 def _filterfetchpaths(repo, paths):
215 def _filterfetchpaths(repo, paths):
213 """return a subset of paths whose history is long and need to fetch linelog
216 """return a subset of paths whose history is long and need to fetch linelog
214 from the server. works with remotefilelog and non-remotefilelog repos.
217 from the server. works with remotefilelog and non-remotefilelog repos.
215 """
218 """
216 threshold = repo.ui.configint(b'fastannotate', b'clientfetchthreshold', 10)
219 threshold = repo.ui.configint(b'fastannotate', b'clientfetchthreshold', 10)
217 if threshold <= 0:
220 if threshold <= 0:
218 return paths
221 return paths
219
222
220 result = []
223 result = []
221 for path in paths:
224 for path in paths:
222 try:
225 try:
223 if len(repo.file(path)) >= threshold:
226 if len(repo.file(path)) >= threshold:
224 result.append(path)
227 result.append(path)
225 except Exception: # file not found etc.
228 except Exception: # file not found etc.
226 result.append(path)
229 result.append(path)
227
230
228 return result
231 return result
229
232
230
233
231 def localreposetup(ui, repo):
234 def localreposetup(ui, repo):
232 class fastannotaterepo(repo.__class__):
235 class fastannotaterepo(repo.__class__):
233 def prefetchfastannotate(self, paths, peer=None):
236 def prefetchfastannotate(self, paths, peer=None):
234 master = _getmaster(self.ui)
237 master = _getmaster(self.ui)
235 needupdatepaths = []
238 needupdatepaths = []
236 lastnodemap = {}
239 lastnodemap = {}
237 try:
240 try:
238 for path in _filterfetchpaths(self, paths):
241 for path in _filterfetchpaths(self, paths):
239 with context.annotatecontext(self, path) as actx:
242 with context.annotatecontext(self, path) as actx:
240 if not actx.isuptodate(master, strict=False):
243 if not actx.isuptodate(master, strict=False):
241 needupdatepaths.append(path)
244 needupdatepaths.append(path)
242 lastnodemap[path] = actx.lastnode
245 lastnodemap[path] = actx.lastnode
243 if needupdatepaths:
246 if needupdatepaths:
244 clientfetch(self, needupdatepaths, lastnodemap, peer)
247 clientfetch(self, needupdatepaths, lastnodemap, peer)
245 except Exception as ex:
248 except Exception as ex:
246 # could be directory not writable or so, not fatal
249 # could be directory not writable or so, not fatal
247 self.ui.debug(b'fastannotate: prefetch failed: %r\n' % ex)
250 self.ui.debug(b'fastannotate: prefetch failed: %r\n' % ex)
248
251
249 repo.__class__ = fastannotaterepo
252 repo.__class__ = fastannotaterepo
250
253
251
254
252 def clientreposetup(ui, repo):
255 def clientreposetup(ui, repo):
253 _registerwireprotocommand()
256 _registerwireprotocommand()
254 if repo.local():
257 if repo.local():
255 localreposetup(ui, repo)
258 localreposetup(ui, repo)
256 # TODO: this mutates global state, but only if at least one repo
259 # TODO: this mutates global state, but only if at least one repo
257 # has the extension enabled. This is probably bad for hgweb.
260 # has the extension enabled. This is probably bad for hgweb.
258 if peersetup not in hg.wirepeersetupfuncs:
261 if peersetup not in hg.wirepeersetupfuncs:
259 hg.wirepeersetupfuncs.append(peersetup)
262 hg.wirepeersetupfuncs.append(peersetup)
General Comments 0
You need to be logged in to leave comments. Login now