Show More
@@ -1,491 +1,491 b'' | |||||
1 | # remotefilectx.py - filectx/workingfilectx implementations for remotefilelog |
|
1 | # remotefilectx.py - filectx/workingfilectx implementations for remotefilelog | |
2 | # |
|
2 | # | |
3 | # Copyright 2013 Facebook, Inc. |
|
3 | # Copyright 2013 Facebook, Inc. | |
4 | # |
|
4 | # | |
5 | # This software may be used and distributed according to the terms of the |
|
5 | # This software may be used and distributed according to the terms of the | |
6 | # GNU General Public License version 2 or any later version. |
|
6 | # GNU General Public License version 2 or any later version. | |
7 | from __future__ import absolute_import |
|
7 | from __future__ import absolute_import | |
8 |
|
8 | |||
9 | import collections |
|
9 | import collections | |
10 | import time |
|
10 | import time | |
11 |
|
11 | |||
12 | from mercurial.node import bin, hex, nullid, nullrev |
|
12 | from mercurial.node import bin, hex, nullid, nullrev | |
13 | from mercurial import ( |
|
13 | from mercurial import ( | |
14 | ancestor, |
|
14 | ancestor, | |
15 | context, |
|
15 | context, | |
16 | error, |
|
16 | error, | |
17 | phases, |
|
17 | phases, | |
18 | pycompat, |
|
18 | pycompat, | |
19 | util, |
|
19 | util, | |
20 | ) |
|
20 | ) | |
21 | from . import shallowutil |
|
21 | from . import shallowutil | |
22 |
|
22 | |||
23 | propertycache = util.propertycache |
|
23 | propertycache = util.propertycache | |
24 | FASTLOG_TIMEOUT_IN_SECS = 0.5 |
|
24 | FASTLOG_TIMEOUT_IN_SECS = 0.5 | |
25 |
|
25 | |||
26 | class remotefilectx(context.filectx): |
|
26 | class remotefilectx(context.filectx): | |
27 | def __init__(self, repo, path, changeid=None, fileid=None, |
|
27 | def __init__(self, repo, path, changeid=None, fileid=None, | |
28 | filelog=None, changectx=None, ancestormap=None): |
|
28 | filelog=None, changectx=None, ancestormap=None): | |
29 | if fileid == nullrev: |
|
29 | if fileid == nullrev: | |
30 | fileid = nullid |
|
30 | fileid = nullid | |
31 | if fileid and len(fileid) == 40: |
|
31 | if fileid and len(fileid) == 40: | |
32 | fileid = bin(fileid) |
|
32 | fileid = bin(fileid) | |
33 | super(remotefilectx, self).__init__(repo, path, changeid, |
|
33 | super(remotefilectx, self).__init__(repo, path, changeid, | |
34 | fileid, filelog, changectx) |
|
34 | fileid, filelog, changectx) | |
35 | self._ancestormap = ancestormap |
|
35 | self._ancestormap = ancestormap | |
36 |
|
36 | |||
37 | def size(self): |
|
37 | def size(self): | |
38 | return self._filelog.size(self._filenode) |
|
38 | return self._filelog.size(self._filenode) | |
39 |
|
39 | |||
40 | @propertycache |
|
40 | @propertycache | |
41 | def _changeid(self): |
|
41 | def _changeid(self): | |
42 | if '_changeid' in self.__dict__: |
|
42 | if '_changeid' in self.__dict__: | |
43 | return self._changeid |
|
43 | return self._changeid | |
44 | elif '_changectx' in self.__dict__: |
|
44 | elif '_changectx' in self.__dict__: | |
45 | return self._changectx.rev() |
|
45 | return self._changectx.rev() | |
46 | elif '_descendantrev' in self.__dict__: |
|
46 | elif '_descendantrev' in self.__dict__: | |
47 | # this file context was created from a revision with a known |
|
47 | # this file context was created from a revision with a known | |
48 | # descendant, we can (lazily) correct for linkrev aliases |
|
48 | # descendant, we can (lazily) correct for linkrev aliases | |
49 | linknode = self._adjustlinknode(self._path, self._filelog, |
|
49 | linknode = self._adjustlinknode(self._path, self._filelog, | |
50 | self._filenode, self._descendantrev) |
|
50 | self._filenode, self._descendantrev) | |
51 | return self._repo.unfiltered().changelog.rev(linknode) |
|
51 | return self._repo.unfiltered().changelog.rev(linknode) | |
52 | else: |
|
52 | else: | |
53 | return self.linkrev() |
|
53 | return self.linkrev() | |
54 |
|
54 | |||
55 | def filectx(self, fileid, changeid=None): |
|
55 | def filectx(self, fileid, changeid=None): | |
56 | '''opens an arbitrary revision of the file without |
|
56 | '''opens an arbitrary revision of the file without | |
57 | opening a new filelog''' |
|
57 | opening a new filelog''' | |
58 | return remotefilectx(self._repo, self._path, fileid=fileid, |
|
58 | return remotefilectx(self._repo, self._path, fileid=fileid, | |
59 | filelog=self._filelog, changeid=changeid) |
|
59 | filelog=self._filelog, changeid=changeid) | |
60 |
|
60 | |||
61 | def linkrev(self): |
|
61 | def linkrev(self): | |
62 | return self._linkrev |
|
62 | return self._linkrev | |
63 |
|
63 | |||
64 | @propertycache |
|
64 | @propertycache | |
65 | def _linkrev(self): |
|
65 | def _linkrev(self): | |
66 | if self._filenode == nullid: |
|
66 | if self._filenode == nullid: | |
67 | return nullrev |
|
67 | return nullrev | |
68 |
|
68 | |||
69 | ancestormap = self.ancestormap() |
|
69 | ancestormap = self.ancestormap() | |
70 | p1, p2, linknode, copyfrom = ancestormap[self._filenode] |
|
70 | p1, p2, linknode, copyfrom = ancestormap[self._filenode] | |
71 | rev = self._repo.changelog.nodemap.get(linknode) |
|
71 | rev = self._repo.changelog.nodemap.get(linknode) | |
72 | if rev is not None: |
|
72 | if rev is not None: | |
73 | return rev |
|
73 | return rev | |
74 |
|
74 | |||
75 | # Search all commits for the appropriate linkrev (slow, but uncommon) |
|
75 | # Search all commits for the appropriate linkrev (slow, but uncommon) | |
76 | path = self._path |
|
76 | path = self._path | |
77 | fileid = self._filenode |
|
77 | fileid = self._filenode | |
78 | cl = self._repo.unfiltered().changelog |
|
78 | cl = self._repo.unfiltered().changelog | |
79 | mfl = self._repo.manifestlog |
|
79 | mfl = self._repo.manifestlog | |
80 |
|
80 | |||
81 | for rev in range(len(cl) - 1, 0, -1): |
|
81 | for rev in range(len(cl) - 1, 0, -1): | |
82 | node = cl.node(rev) |
|
82 | node = cl.node(rev) | |
83 | data = cl.read(node) # get changeset data (we avoid object creation) |
|
83 | data = cl.read(node) # get changeset data (we avoid object creation) | |
84 | if path in data[3]: # checking the 'files' field. |
|
84 | if path in data[3]: # checking the 'files' field. | |
85 | # The file has been touched, check if the hash is what we're |
|
85 | # The file has been touched, check if the hash is what we're | |
86 | # looking for. |
|
86 | # looking for. | |
87 | if fileid == mfl[data[0]].readfast().get(path): |
|
87 | if fileid == mfl[data[0]].readfast().get(path): | |
88 | return rev |
|
88 | return rev | |
89 |
|
89 | |||
90 | # Couldn't find the linkrev. This should generally not happen, and will |
|
90 | # Couldn't find the linkrev. This should generally not happen, and will | |
91 | # likely cause a crash. |
|
91 | # likely cause a crash. | |
92 | return None |
|
92 | return None | |
93 |
|
93 | |||
94 | def introrev(self): |
|
94 | def introrev(self): | |
95 | """return the rev of the changeset which introduced this file revision |
|
95 | """return the rev of the changeset which introduced this file revision | |
96 |
|
96 | |||
97 | This method is different from linkrev because it take into account the |
|
97 | This method is different from linkrev because it take into account the | |
98 | changeset the filectx was created from. It ensures the returned |
|
98 | changeset the filectx was created from. It ensures the returned | |
99 | revision is one of its ancestors. This prevents bugs from |
|
99 | revision is one of its ancestors. This prevents bugs from | |
100 | 'linkrev-shadowing' when a file revision is used by multiple |
|
100 | 'linkrev-shadowing' when a file revision is used by multiple | |
101 | changesets. |
|
101 | changesets. | |
102 | """ |
|
102 | """ | |
103 | lkr = self.linkrev() |
|
103 | lkr = self.linkrev() | |
104 | attrs = vars(self) |
|
104 | attrs = vars(self) | |
105 | noctx = not ('_changeid' in attrs or '_changectx' in attrs) |
|
105 | noctx = not ('_changeid' in attrs or '_changectx' in attrs) | |
106 | if noctx or self.rev() == lkr: |
|
106 | if noctx or self.rev() == lkr: | |
107 | return lkr |
|
107 | return lkr | |
108 | linknode = self._adjustlinknode(self._path, self._filelog, |
|
108 | linknode = self._adjustlinknode(self._path, self._filelog, | |
109 | self._filenode, self.rev(), |
|
109 | self._filenode, self.rev(), | |
110 | inclusive=True) |
|
110 | inclusive=True) | |
111 | return self._repo.changelog.rev(linknode) |
|
111 | return self._repo.changelog.rev(linknode) | |
112 |
|
112 | |||
113 | def renamed(self): |
|
113 | def renamed(self): | |
114 | """check if file was actually renamed in this changeset revision |
|
114 | """check if file was actually renamed in this changeset revision | |
115 |
|
115 | |||
116 | If rename logged in file revision, we report copy for changeset only |
|
116 | If rename logged in file revision, we report copy for changeset only | |
117 | if file revisions linkrev points back to the changeset in question |
|
117 | if file revisions linkrev points back to the changeset in question | |
118 | or both changeset parents contain different file revisions. |
|
118 | or both changeset parents contain different file revisions. | |
119 | """ |
|
119 | """ | |
120 | ancestormap = self.ancestormap() |
|
120 | ancestormap = self.ancestormap() | |
121 |
|
121 | |||
122 | p1, p2, linknode, copyfrom = ancestormap[self._filenode] |
|
122 | p1, p2, linknode, copyfrom = ancestormap[self._filenode] | |
123 | if not copyfrom: |
|
123 | if not copyfrom: | |
124 | return None |
|
124 | return None | |
125 |
|
125 | |||
126 | renamed = (copyfrom, p1) |
|
126 | renamed = (copyfrom, p1) | |
127 | if self.rev() == self.linkrev(): |
|
127 | if self.rev() == self.linkrev(): | |
128 | return renamed |
|
128 | return renamed | |
129 |
|
129 | |||
130 | name = self.path() |
|
130 | name = self.path() | |
131 | fnode = self._filenode |
|
131 | fnode = self._filenode | |
132 | for p in self._changectx.parents(): |
|
132 | for p in self._changectx.parents(): | |
133 | try: |
|
133 | try: | |
134 | if fnode == p.filenode(name): |
|
134 | if fnode == p.filenode(name): | |
135 | return None |
|
135 | return None | |
136 | except error.LookupError: |
|
136 | except error.LookupError: | |
137 | pass |
|
137 | pass | |
138 | return renamed |
|
138 | return renamed | |
139 |
|
139 | |||
140 | def ancestormap(self): |
|
140 | def ancestormap(self): | |
141 | if not self._ancestormap: |
|
141 | if not self._ancestormap: | |
142 | self._ancestormap = self.filelog().ancestormap(self._filenode) |
|
142 | self._ancestormap = self.filelog().ancestormap(self._filenode) | |
143 |
|
143 | |||
144 | return self._ancestormap |
|
144 | return self._ancestormap | |
145 |
|
145 | |||
146 | def parents(self): |
|
146 | def parents(self): | |
147 | repo = self._repo |
|
147 | repo = self._repo | |
148 | ancestormap = self.ancestormap() |
|
148 | ancestormap = self.ancestormap() | |
149 |
|
149 | |||
150 | p1, p2, linknode, copyfrom = ancestormap[self._filenode] |
|
150 | p1, p2, linknode, copyfrom = ancestormap[self._filenode] | |
151 | results = [] |
|
151 | results = [] | |
152 | if p1 != nullid: |
|
152 | if p1 != nullid: | |
153 | path = copyfrom or self._path |
|
153 | path = copyfrom or self._path | |
154 | flog = repo.file(path) |
|
154 | flog = repo.file(path) | |
155 | p1ctx = remotefilectx(repo, path, fileid=p1, filelog=flog, |
|
155 | p1ctx = remotefilectx(repo, path, fileid=p1, filelog=flog, | |
156 | ancestormap=ancestormap) |
|
156 | ancestormap=ancestormap) | |
157 | p1ctx._descendantrev = self.rev() |
|
157 | p1ctx._descendantrev = self.rev() | |
158 | results.append(p1ctx) |
|
158 | results.append(p1ctx) | |
159 |
|
159 | |||
160 | if p2 != nullid: |
|
160 | if p2 != nullid: | |
161 | path = self._path |
|
161 | path = self._path | |
162 | flog = repo.file(path) |
|
162 | flog = repo.file(path) | |
163 | p2ctx = remotefilectx(repo, path, fileid=p2, filelog=flog, |
|
163 | p2ctx = remotefilectx(repo, path, fileid=p2, filelog=flog, | |
164 | ancestormap=ancestormap) |
|
164 | ancestormap=ancestormap) | |
165 | p2ctx._descendantrev = self.rev() |
|
165 | p2ctx._descendantrev = self.rev() | |
166 | results.append(p2ctx) |
|
166 | results.append(p2ctx) | |
167 |
|
167 | |||
168 | return results |
|
168 | return results | |
169 |
|
169 | |||
170 | def _nodefromancrev(self, ancrev, cl, mfl, path, fnode): |
|
170 | def _nodefromancrev(self, ancrev, cl, mfl, path, fnode): | |
171 | """returns the node for <path> in <ancrev> if content matches <fnode>""" |
|
171 | """returns the node for <path> in <ancrev> if content matches <fnode>""" | |
172 | ancctx = cl.read(ancrev) # This avoids object creation. |
|
172 | ancctx = cl.read(ancrev) # This avoids object creation. | |
173 | manifestnode, files = ancctx[0], ancctx[3] |
|
173 | manifestnode, files = ancctx[0], ancctx[3] | |
174 | # If the file was touched in this ancestor, and the content is similar |
|
174 | # If the file was touched in this ancestor, and the content is similar | |
175 | # to the one we are searching for. |
|
175 | # to the one we are searching for. | |
176 | if path in files and fnode == mfl[manifestnode].readfast().get(path): |
|
176 | if path in files and fnode == mfl[manifestnode].readfast().get(path): | |
177 | return cl.node(ancrev) |
|
177 | return cl.node(ancrev) | |
178 | return None |
|
178 | return None | |
179 |
|
179 | |||
180 | def _adjustlinknode(self, path, filelog, fnode, srcrev, inclusive=False): |
|
180 | def _adjustlinknode(self, path, filelog, fnode, srcrev, inclusive=False): | |
181 | """return the first ancestor of <srcrev> introducing <fnode> |
|
181 | """return the first ancestor of <srcrev> introducing <fnode> | |
182 |
|
182 | |||
183 | If the linkrev of the file revision does not point to an ancestor of |
|
183 | If the linkrev of the file revision does not point to an ancestor of | |
184 | srcrev, we'll walk down the ancestors until we find one introducing |
|
184 | srcrev, we'll walk down the ancestors until we find one introducing | |
185 | this file revision. |
|
185 | this file revision. | |
186 |
|
186 | |||
187 | :repo: a localrepository object (used to access changelog and manifest) |
|
187 | :repo: a localrepository object (used to access changelog and manifest) | |
188 | :path: the file path |
|
188 | :path: the file path | |
189 | :fnode: the nodeid of the file revision |
|
189 | :fnode: the nodeid of the file revision | |
190 | :filelog: the filelog of this path |
|
190 | :filelog: the filelog of this path | |
191 | :srcrev: the changeset revision we search ancestors from |
|
191 | :srcrev: the changeset revision we search ancestors from | |
192 | :inclusive: if true, the src revision will also be checked |
|
192 | :inclusive: if true, the src revision will also be checked | |
193 |
|
193 | |||
194 | Note: This is based on adjustlinkrev in core, but it's quite different. |
|
194 | Note: This is based on adjustlinkrev in core, but it's quite different. | |
195 |
|
195 | |||
196 | adjustlinkrev depends on the fact that the linkrev is the bottom most |
|
196 | adjustlinkrev depends on the fact that the linkrev is the bottom most | |
197 | node, and uses that as a stopping point for the ancestor traversal. We |
|
197 | node, and uses that as a stopping point for the ancestor traversal. We | |
198 | can't do that here because the linknode is not guaranteed to be the |
|
198 | can't do that here because the linknode is not guaranteed to be the | |
199 | bottom most one. |
|
199 | bottom most one. | |
200 |
|
200 | |||
201 | In our code here, we actually know what a bunch of potential ancestor |
|
201 | In our code here, we actually know what a bunch of potential ancestor | |
202 | linknodes are, so instead of stopping the cheap-ancestor-traversal when |
|
202 | linknodes are, so instead of stopping the cheap-ancestor-traversal when | |
203 | we get to a linkrev, we stop when we see any of the known linknodes. |
|
203 | we get to a linkrev, we stop when we see any of the known linknodes. | |
204 | """ |
|
204 | """ | |
205 | repo = self._repo |
|
205 | repo = self._repo | |
206 | cl = repo.unfiltered().changelog |
|
206 | cl = repo.unfiltered().changelog | |
207 | mfl = repo.manifestlog |
|
207 | mfl = repo.manifestlog | |
208 | ancestormap = self.ancestormap() |
|
208 | ancestormap = self.ancestormap() | |
209 | linknode = ancestormap[fnode][2] |
|
209 | linknode = ancestormap[fnode][2] | |
210 |
|
210 | |||
211 | if srcrev is None: |
|
211 | if srcrev is None: | |
212 | # wctx case, used by workingfilectx during mergecopy |
|
212 | # wctx case, used by workingfilectx during mergecopy | |
213 | revs = [p.rev() for p in self._repo[None].parents()] |
|
213 | revs = [p.rev() for p in self._repo[None].parents()] | |
214 | inclusive = True # we skipped the real (revless) source |
|
214 | inclusive = True # we skipped the real (revless) source | |
215 | else: |
|
215 | else: | |
216 | revs = [srcrev] |
|
216 | revs = [srcrev] | |
217 |
|
217 | |||
218 | if self._verifylinknode(revs, linknode): |
|
218 | if self._verifylinknode(revs, linknode): | |
219 | return linknode |
|
219 | return linknode | |
220 |
|
220 | |||
221 | commonlogkwargs = { |
|
221 | commonlogkwargs = { | |
222 | r'revs': ' '.join([hex(cl.node(rev)) for rev in revs]), |
|
222 | r'revs': ' '.join([hex(cl.node(rev)) for rev in revs]), | |
223 | r'fnode': hex(fnode), |
|
223 | r'fnode': hex(fnode), | |
224 | r'filepath': path, |
|
224 | r'filepath': path, | |
225 | r'user': shallowutil.getusername(repo.ui), |
|
225 | r'user': shallowutil.getusername(repo.ui), | |
226 | r'reponame': shallowutil.getreponame(repo.ui), |
|
226 | r'reponame': shallowutil.getreponame(repo.ui), | |
227 | } |
|
227 | } | |
228 |
|
228 | |||
229 | repo.ui.log('linkrevfixup', 'adjusting linknode', **commonlogkwargs) |
|
229 | repo.ui.log('linkrevfixup', 'adjusting linknode\n', **commonlogkwargs) | |
230 |
|
230 | |||
231 | pc = repo._phasecache |
|
231 | pc = repo._phasecache | |
232 | seenpublic = False |
|
232 | seenpublic = False | |
233 | iteranc = cl.ancestors(revs, inclusive=inclusive) |
|
233 | iteranc = cl.ancestors(revs, inclusive=inclusive) | |
234 | for ancrev in iteranc: |
|
234 | for ancrev in iteranc: | |
235 | # First, check locally-available history. |
|
235 | # First, check locally-available history. | |
236 | lnode = self._nodefromancrev(ancrev, cl, mfl, path, fnode) |
|
236 | lnode = self._nodefromancrev(ancrev, cl, mfl, path, fnode) | |
237 | if lnode is not None: |
|
237 | if lnode is not None: | |
238 | return lnode |
|
238 | return lnode | |
239 |
|
239 | |||
240 | # adjusting linknode can be super-slow. To mitigate the issue |
|
240 | # adjusting linknode can be super-slow. To mitigate the issue | |
241 | # we use two heuristics: calling fastlog and forcing remotefilelog |
|
241 | # we use two heuristics: calling fastlog and forcing remotefilelog | |
242 | # prefetch |
|
242 | # prefetch | |
243 | if not seenpublic and pc.phase(repo, ancrev) == phases.public: |
|
243 | if not seenpublic and pc.phase(repo, ancrev) == phases.public: | |
244 | # TODO: there used to be a codepath to fetch linknodes |
|
244 | # TODO: there used to be a codepath to fetch linknodes | |
245 | # from a server as a fast path, but it appeared to |
|
245 | # from a server as a fast path, but it appeared to | |
246 | # depend on an API FB added to their phabricator. |
|
246 | # depend on an API FB added to their phabricator. | |
247 | lnode = self._forceprefetch(repo, path, fnode, revs, |
|
247 | lnode = self._forceprefetch(repo, path, fnode, revs, | |
248 | commonlogkwargs) |
|
248 | commonlogkwargs) | |
249 | if lnode: |
|
249 | if lnode: | |
250 | return lnode |
|
250 | return lnode | |
251 | seenpublic = True |
|
251 | seenpublic = True | |
252 |
|
252 | |||
253 | return linknode |
|
253 | return linknode | |
254 |
|
254 | |||
255 | def _forceprefetch(self, repo, path, fnode, revs, |
|
255 | def _forceprefetch(self, repo, path, fnode, revs, | |
256 | commonlogkwargs): |
|
256 | commonlogkwargs): | |
257 | # This next part is super non-obvious, so big comment block time! |
|
257 | # This next part is super non-obvious, so big comment block time! | |
258 | # |
|
258 | # | |
259 | # It is possible to get extremely bad performance here when a fairly |
|
259 | # It is possible to get extremely bad performance here when a fairly | |
260 | # common set of circumstances occur when this extension is combined |
|
260 | # common set of circumstances occur when this extension is combined | |
261 | # with a server-side commit rewriting extension like pushrebase. |
|
261 | # with a server-side commit rewriting extension like pushrebase. | |
262 | # |
|
262 | # | |
263 | # First, an engineer creates Commit A and pushes it to the server. |
|
263 | # First, an engineer creates Commit A and pushes it to the server. | |
264 | # While the server's data structure will have the correct linkrev |
|
264 | # While the server's data structure will have the correct linkrev | |
265 | # for the files touched in Commit A, the client will have the |
|
265 | # for the files touched in Commit A, the client will have the | |
266 | # linkrev of the local commit, which is "invalid" because it's not |
|
266 | # linkrev of the local commit, which is "invalid" because it's not | |
267 | # an ancestor of the main line of development. |
|
267 | # an ancestor of the main line of development. | |
268 | # |
|
268 | # | |
269 | # The client will never download the remotefilelog with the correct |
|
269 | # The client will never download the remotefilelog with the correct | |
270 | # linkrev as long as nobody else touches that file, since the file |
|
270 | # linkrev as long as nobody else touches that file, since the file | |
271 | # data and history hasn't changed since Commit A. |
|
271 | # data and history hasn't changed since Commit A. | |
272 | # |
|
272 | # | |
273 | # After a long time (or a short time in a heavily used repo), if the |
|
273 | # After a long time (or a short time in a heavily used repo), if the | |
274 | # same engineer returns to change the same file, some commands -- |
|
274 | # same engineer returns to change the same file, some commands -- | |
275 | # such as amends of commits with file moves, logs, diffs, etc -- |
|
275 | # such as amends of commits with file moves, logs, diffs, etc -- | |
276 | # can trigger this _adjustlinknode code. In those cases, finding |
|
276 | # can trigger this _adjustlinknode code. In those cases, finding | |
277 | # the correct rev can become quite expensive, as the correct |
|
277 | # the correct rev can become quite expensive, as the correct | |
278 | # revision is far back in history and we need to walk back through |
|
278 | # revision is far back in history and we need to walk back through | |
279 | # history to find it. |
|
279 | # history to find it. | |
280 | # |
|
280 | # | |
281 | # In order to improve this situation, we force a prefetch of the |
|
281 | # In order to improve this situation, we force a prefetch of the | |
282 | # remotefilelog data blob for the file we were called on. We do this |
|
282 | # remotefilelog data blob for the file we were called on. We do this | |
283 | # at most once, when we first see a public commit in the history we |
|
283 | # at most once, when we first see a public commit in the history we | |
284 | # are traversing. |
|
284 | # are traversing. | |
285 | # |
|
285 | # | |
286 | # Forcing the prefetch means we will download the remote blob even |
|
286 | # Forcing the prefetch means we will download the remote blob even | |
287 | # if we have the "correct" blob in the local store. Since the union |
|
287 | # if we have the "correct" blob in the local store. Since the union | |
288 | # store checks the remote store first, this means we are much more |
|
288 | # store checks the remote store first, this means we are much more | |
289 | # likely to get the correct linkrev at this point. |
|
289 | # likely to get the correct linkrev at this point. | |
290 | # |
|
290 | # | |
291 | # In rare circumstances (such as the server having a suboptimal |
|
291 | # In rare circumstances (such as the server having a suboptimal | |
292 | # linkrev for our use case), we will fall back to the old slow path. |
|
292 | # linkrev for our use case), we will fall back to the old slow path. | |
293 | # |
|
293 | # | |
294 | # We may want to add additional heuristics here in the future if |
|
294 | # We may want to add additional heuristics here in the future if | |
295 | # the slow path is used too much. One promising possibility is using |
|
295 | # the slow path is used too much. One promising possibility is using | |
296 | # obsolescence markers to find a more-likely-correct linkrev. |
|
296 | # obsolescence markers to find a more-likely-correct linkrev. | |
297 |
|
297 | |||
298 | logmsg = '' |
|
298 | logmsg = '' | |
299 | start = time.time() |
|
299 | start = time.time() | |
300 | try: |
|
300 | try: | |
301 | repo.fileservice.prefetch([(path, hex(fnode))], force=True) |
|
301 | repo.fileservice.prefetch([(path, hex(fnode))], force=True) | |
302 |
|
302 | |||
303 | # Now that we've downloaded a new blob from the server, |
|
303 | # Now that we've downloaded a new blob from the server, | |
304 | # we need to rebuild the ancestor map to recompute the |
|
304 | # we need to rebuild the ancestor map to recompute the | |
305 | # linknodes. |
|
305 | # linknodes. | |
306 | self._ancestormap = None |
|
306 | self._ancestormap = None | |
307 | linknode = self.ancestormap()[fnode][2] # 2 is linknode |
|
307 | linknode = self.ancestormap()[fnode][2] # 2 is linknode | |
308 | if self._verifylinknode(revs, linknode): |
|
308 | if self._verifylinknode(revs, linknode): | |
309 | logmsg = 'remotefilelog prefetching succeeded' |
|
309 | logmsg = 'remotefilelog prefetching succeeded' | |
310 | return linknode |
|
310 | return linknode | |
311 | logmsg = 'remotefilelog prefetching not found' |
|
311 | logmsg = 'remotefilelog prefetching not found' | |
312 | return None |
|
312 | return None | |
313 | except Exception as e: |
|
313 | except Exception as e: | |
314 | logmsg = 'remotefilelog prefetching failed (%s)' % e |
|
314 | logmsg = 'remotefilelog prefetching failed (%s)' % e | |
315 | return None |
|
315 | return None | |
316 | finally: |
|
316 | finally: | |
317 | elapsed = time.time() - start |
|
317 | elapsed = time.time() - start | |
318 | repo.ui.log('linkrevfixup', logmsg, elapsed=elapsed * 1000, |
|
318 | repo.ui.log('linkrevfixup', logmsg + '\n', elapsed=elapsed * 1000, | |
319 | **pycompat.strkwargs(commonlogkwargs)) |
|
319 | **pycompat.strkwargs(commonlogkwargs)) | |
320 |
|
320 | |||
321 | def _verifylinknode(self, revs, linknode): |
|
321 | def _verifylinknode(self, revs, linknode): | |
322 | """ |
|
322 | """ | |
323 | Check if a linknode is correct one for the current history. |
|
323 | Check if a linknode is correct one for the current history. | |
324 |
|
324 | |||
325 | That is, return True if the linkrev is the ancestor of any of the |
|
325 | That is, return True if the linkrev is the ancestor of any of the | |
326 | passed in revs, otherwise return False. |
|
326 | passed in revs, otherwise return False. | |
327 |
|
327 | |||
328 | `revs` is a list that usually has one element -- usually the wdir parent |
|
328 | `revs` is a list that usually has one element -- usually the wdir parent | |
329 | or the user-passed rev we're looking back from. It may contain two revs |
|
329 | or the user-passed rev we're looking back from. It may contain two revs | |
330 | when there is a merge going on, or zero revs when a root node with no |
|
330 | when there is a merge going on, or zero revs when a root node with no | |
331 | parents is being created. |
|
331 | parents is being created. | |
332 | """ |
|
332 | """ | |
333 | if not revs: |
|
333 | if not revs: | |
334 | return False |
|
334 | return False | |
335 | try: |
|
335 | try: | |
336 | # Use the C fastpath to check if the given linknode is correct. |
|
336 | # Use the C fastpath to check if the given linknode is correct. | |
337 | cl = self._repo.unfiltered().changelog |
|
337 | cl = self._repo.unfiltered().changelog | |
338 | return any(cl.isancestor(linknode, cl.node(r)) for r in revs) |
|
338 | return any(cl.isancestor(linknode, cl.node(r)) for r in revs) | |
339 | except error.LookupError: |
|
339 | except error.LookupError: | |
340 | # The linknode read from the blob may have been stripped or |
|
340 | # The linknode read from the blob may have been stripped or | |
341 | # otherwise not present in the repository anymore. Do not fail hard |
|
341 | # otherwise not present in the repository anymore. Do not fail hard | |
342 | # in this case. Instead, return false and continue the search for |
|
342 | # in this case. Instead, return false and continue the search for | |
343 | # the correct linknode. |
|
343 | # the correct linknode. | |
344 | return False |
|
344 | return False | |
345 |
|
345 | |||
346 | def ancestors(self, followfirst=False): |
|
346 | def ancestors(self, followfirst=False): | |
347 | ancestors = [] |
|
347 | ancestors = [] | |
348 | queue = collections.deque((self,)) |
|
348 | queue = collections.deque((self,)) | |
349 | seen = set() |
|
349 | seen = set() | |
350 | while queue: |
|
350 | while queue: | |
351 | current = queue.pop() |
|
351 | current = queue.pop() | |
352 | if current.filenode() in seen: |
|
352 | if current.filenode() in seen: | |
353 | continue |
|
353 | continue | |
354 | seen.add(current.filenode()) |
|
354 | seen.add(current.filenode()) | |
355 |
|
355 | |||
356 | ancestors.append(current) |
|
356 | ancestors.append(current) | |
357 |
|
357 | |||
358 | parents = current.parents() |
|
358 | parents = current.parents() | |
359 | first = True |
|
359 | first = True | |
360 | for p in parents: |
|
360 | for p in parents: | |
361 | if first or not followfirst: |
|
361 | if first or not followfirst: | |
362 | queue.append(p) |
|
362 | queue.append(p) | |
363 | first = False |
|
363 | first = False | |
364 |
|
364 | |||
365 | # Remove self |
|
365 | # Remove self | |
366 | ancestors.pop(0) |
|
366 | ancestors.pop(0) | |
367 |
|
367 | |||
368 | # Sort by linkrev |
|
368 | # Sort by linkrev | |
369 | # The copy tracing algorithm depends on these coming out in order |
|
369 | # The copy tracing algorithm depends on these coming out in order | |
370 | ancestors = sorted(ancestors, reverse=True, key=lambda x:x.linkrev()) |
|
370 | ancestors = sorted(ancestors, reverse=True, key=lambda x:x.linkrev()) | |
371 |
|
371 | |||
372 | for ancestor in ancestors: |
|
372 | for ancestor in ancestors: | |
373 | yield ancestor |
|
373 | yield ancestor | |
374 |
|
374 | |||
375 | def ancestor(self, fc2, actx): |
|
375 | def ancestor(self, fc2, actx): | |
376 | # the easy case: no (relevant) renames |
|
376 | # the easy case: no (relevant) renames | |
377 | if fc2.path() == self.path() and self.path() in actx: |
|
377 | if fc2.path() == self.path() and self.path() in actx: | |
378 | return actx[self.path()] |
|
378 | return actx[self.path()] | |
379 |
|
379 | |||
380 | # the next easiest cases: unambiguous predecessor (name trumps |
|
380 | # the next easiest cases: unambiguous predecessor (name trumps | |
381 | # history) |
|
381 | # history) | |
382 | if self.path() in actx and fc2.path() not in actx: |
|
382 | if self.path() in actx and fc2.path() not in actx: | |
383 | return actx[self.path()] |
|
383 | return actx[self.path()] | |
384 | if fc2.path() in actx and self.path() not in actx: |
|
384 | if fc2.path() in actx and self.path() not in actx: | |
385 | return actx[fc2.path()] |
|
385 | return actx[fc2.path()] | |
386 |
|
386 | |||
387 | # do a full traversal |
|
387 | # do a full traversal | |
388 | amap = self.ancestormap() |
|
388 | amap = self.ancestormap() | |
389 | bmap = fc2.ancestormap() |
|
389 | bmap = fc2.ancestormap() | |
390 |
|
390 | |||
391 | def parents(x): |
|
391 | def parents(x): | |
392 | f, n = x |
|
392 | f, n = x | |
393 | p = amap.get(n) or bmap.get(n) |
|
393 | p = amap.get(n) or bmap.get(n) | |
394 | if not p: |
|
394 | if not p: | |
395 | return [] |
|
395 | return [] | |
396 |
|
396 | |||
397 | return [(p[3] or f, p[0]), (f, p[1])] |
|
397 | return [(p[3] or f, p[0]), (f, p[1])] | |
398 |
|
398 | |||
399 | a = (self.path(), self.filenode()) |
|
399 | a = (self.path(), self.filenode()) | |
400 | b = (fc2.path(), fc2.filenode()) |
|
400 | b = (fc2.path(), fc2.filenode()) | |
401 | result = ancestor.genericancestor(a, b, parents) |
|
401 | result = ancestor.genericancestor(a, b, parents) | |
402 | if result: |
|
402 | if result: | |
403 | f, n = result |
|
403 | f, n = result | |
404 | r = remotefilectx(self._repo, f, fileid=n, |
|
404 | r = remotefilectx(self._repo, f, fileid=n, | |
405 | ancestormap=amap) |
|
405 | ancestormap=amap) | |
406 | return r |
|
406 | return r | |
407 |
|
407 | |||
408 | return None |
|
408 | return None | |
409 |
|
409 | |||
410 | def annotate(self, *args, **kwargs): |
|
410 | def annotate(self, *args, **kwargs): | |
411 | introctx = self |
|
411 | introctx = self | |
412 | prefetchskip = kwargs.pop(r'prefetchskip', None) |
|
412 | prefetchskip = kwargs.pop(r'prefetchskip', None) | |
413 | if prefetchskip: |
|
413 | if prefetchskip: | |
414 | # use introrev so prefetchskip can be accurately tested |
|
414 | # use introrev so prefetchskip can be accurately tested | |
415 | introrev = self.introrev() |
|
415 | introrev = self.introrev() | |
416 | if self.rev() != introrev: |
|
416 | if self.rev() != introrev: | |
417 | introctx = remotefilectx(self._repo, self._path, |
|
417 | introctx = remotefilectx(self._repo, self._path, | |
418 | changeid=introrev, |
|
418 | changeid=introrev, | |
419 | fileid=self._filenode, |
|
419 | fileid=self._filenode, | |
420 | filelog=self._filelog, |
|
420 | filelog=self._filelog, | |
421 | ancestormap=self._ancestormap) |
|
421 | ancestormap=self._ancestormap) | |
422 |
|
422 | |||
423 | # like self.ancestors, but append to "fetch" and skip visiting parents |
|
423 | # like self.ancestors, but append to "fetch" and skip visiting parents | |
424 | # of nodes in "prefetchskip". |
|
424 | # of nodes in "prefetchskip". | |
425 | fetch = [] |
|
425 | fetch = [] | |
426 | seen = set() |
|
426 | seen = set() | |
427 | queue = collections.deque((introctx,)) |
|
427 | queue = collections.deque((introctx,)) | |
428 | seen.add(introctx.node()) |
|
428 | seen.add(introctx.node()) | |
429 | while queue: |
|
429 | while queue: | |
430 | current = queue.pop() |
|
430 | current = queue.pop() | |
431 | if current.filenode() != self.filenode(): |
|
431 | if current.filenode() != self.filenode(): | |
432 | # this is a "joint point". fastannotate needs contents of |
|
432 | # this is a "joint point". fastannotate needs contents of | |
433 | # "joint point"s to calculate diffs for side branches. |
|
433 | # "joint point"s to calculate diffs for side branches. | |
434 | fetch.append((current.path(), hex(current.filenode()))) |
|
434 | fetch.append((current.path(), hex(current.filenode()))) | |
435 | if prefetchskip and current in prefetchskip: |
|
435 | if prefetchskip and current in prefetchskip: | |
436 | continue |
|
436 | continue | |
437 | for parent in current.parents(): |
|
437 | for parent in current.parents(): | |
438 | if parent.node() not in seen: |
|
438 | if parent.node() not in seen: | |
439 | seen.add(parent.node()) |
|
439 | seen.add(parent.node()) | |
440 | queue.append(parent) |
|
440 | queue.append(parent) | |
441 |
|
441 | |||
442 | self._repo.ui.debug('remotefilelog: prefetching %d files ' |
|
442 | self._repo.ui.debug('remotefilelog: prefetching %d files ' | |
443 | 'for annotate\n' % len(fetch)) |
|
443 | 'for annotate\n' % len(fetch)) | |
444 | if fetch: |
|
444 | if fetch: | |
445 | self._repo.fileservice.prefetch(fetch) |
|
445 | self._repo.fileservice.prefetch(fetch) | |
446 | return super(remotefilectx, self).annotate(*args, **kwargs) |
|
446 | return super(remotefilectx, self).annotate(*args, **kwargs) | |
447 |
|
447 | |||
448 | # Return empty set so that the hg serve and thg don't stack trace |
|
448 | # Return empty set so that the hg serve and thg don't stack trace | |
449 | def children(self): |
|
449 | def children(self): | |
450 | return [] |
|
450 | return [] | |
451 |
|
451 | |||
452 | class remoteworkingfilectx(context.workingfilectx, remotefilectx): |
|
452 | class remoteworkingfilectx(context.workingfilectx, remotefilectx): | |
453 | def __init__(self, repo, path, filelog=None, workingctx=None): |
|
453 | def __init__(self, repo, path, filelog=None, workingctx=None): | |
454 | self._ancestormap = None |
|
454 | self._ancestormap = None | |
455 | return super(remoteworkingfilectx, self).__init__(repo, path, |
|
455 | return super(remoteworkingfilectx, self).__init__(repo, path, | |
456 | filelog, workingctx) |
|
456 | filelog, workingctx) | |
457 |
|
457 | |||
458 | def parents(self): |
|
458 | def parents(self): | |
459 | return remotefilectx.parents(self) |
|
459 | return remotefilectx.parents(self) | |
460 |
|
460 | |||
461 | def ancestormap(self): |
|
461 | def ancestormap(self): | |
462 | if not self._ancestormap: |
|
462 | if not self._ancestormap: | |
463 | path = self._path |
|
463 | path = self._path | |
464 | pcl = self._changectx._parents |
|
464 | pcl = self._changectx._parents | |
465 | renamed = self.renamed() |
|
465 | renamed = self.renamed() | |
466 |
|
466 | |||
467 | if renamed: |
|
467 | if renamed: | |
468 | p1 = renamed |
|
468 | p1 = renamed | |
469 | else: |
|
469 | else: | |
470 | p1 = (path, pcl[0]._manifest.get(path, nullid)) |
|
470 | p1 = (path, pcl[0]._manifest.get(path, nullid)) | |
471 |
|
471 | |||
472 | p2 = (path, nullid) |
|
472 | p2 = (path, nullid) | |
473 | if len(pcl) > 1: |
|
473 | if len(pcl) > 1: | |
474 | p2 = (path, pcl[1]._manifest.get(path, nullid)) |
|
474 | p2 = (path, pcl[1]._manifest.get(path, nullid)) | |
475 |
|
475 | |||
476 | m = {} |
|
476 | m = {} | |
477 | if p1[1] != nullid: |
|
477 | if p1[1] != nullid: | |
478 | p1ctx = self._repo.filectx(p1[0], fileid=p1[1]) |
|
478 | p1ctx = self._repo.filectx(p1[0], fileid=p1[1]) | |
479 | m.update(p1ctx.filelog().ancestormap(p1[1])) |
|
479 | m.update(p1ctx.filelog().ancestormap(p1[1])) | |
480 |
|
480 | |||
481 | if p2[1] != nullid: |
|
481 | if p2[1] != nullid: | |
482 | p2ctx = self._repo.filectx(p2[0], fileid=p2[1]) |
|
482 | p2ctx = self._repo.filectx(p2[0], fileid=p2[1]) | |
483 | m.update(p2ctx.filelog().ancestormap(p2[1])) |
|
483 | m.update(p2ctx.filelog().ancestormap(p2[1])) | |
484 |
|
484 | |||
485 | copyfrom = '' |
|
485 | copyfrom = '' | |
486 | if renamed: |
|
486 | if renamed: | |
487 | copyfrom = renamed[0] |
|
487 | copyfrom = renamed[0] | |
488 | m[None] = (p1[1], p2[1], nullid, copyfrom) |
|
488 | m[None] = (p1[1], p2[1], nullid, copyfrom) | |
489 | self._ancestormap = m |
|
489 | self._ancestormap = m | |
490 |
|
490 | |||
491 | return self._ancestormap |
|
491 | return self._ancestormap |
@@ -1,492 +1,492 b'' | |||||
1 | # shallowutil.py -- remotefilelog utilities |
|
1 | # shallowutil.py -- remotefilelog utilities | |
2 | # |
|
2 | # | |
3 | # Copyright 2014 Facebook, Inc. |
|
3 | # Copyright 2014 Facebook, Inc. | |
4 | # |
|
4 | # | |
5 | # This software may be used and distributed according to the terms of the |
|
5 | # This software may be used and distributed according to the terms of the | |
6 | # GNU General Public License version 2 or any later version. |
|
6 | # GNU General Public License version 2 or any later version. | |
7 | from __future__ import absolute_import |
|
7 | from __future__ import absolute_import | |
8 |
|
8 | |||
9 | import collections |
|
9 | import collections | |
10 | import errno |
|
10 | import errno | |
11 | import hashlib |
|
11 | import hashlib | |
12 | import os |
|
12 | import os | |
13 | import stat |
|
13 | import stat | |
14 | import struct |
|
14 | import struct | |
15 | import tempfile |
|
15 | import tempfile | |
16 |
|
16 | |||
17 | from mercurial.i18n import _ |
|
17 | from mercurial.i18n import _ | |
18 | from mercurial import ( |
|
18 | from mercurial import ( | |
19 | error, |
|
19 | error, | |
20 | node, |
|
20 | node, | |
21 | pycompat, |
|
21 | pycompat, | |
22 | revlog, |
|
22 | revlog, | |
23 | util, |
|
23 | util, | |
24 | ) |
|
24 | ) | |
25 | from mercurial.utils import ( |
|
25 | from mercurial.utils import ( | |
26 | storageutil, |
|
26 | storageutil, | |
27 | stringutil, |
|
27 | stringutil, | |
28 | ) |
|
28 | ) | |
29 | from . import constants |
|
29 | from . import constants | |
30 |
|
30 | |||
31 | if not pycompat.iswindows: |
|
31 | if not pycompat.iswindows: | |
32 | import grp |
|
32 | import grp | |
33 |
|
33 | |||
34 | def isenabled(repo): |
|
34 | def isenabled(repo): | |
35 | """returns whether the repository is remotefilelog enabled or not""" |
|
35 | """returns whether the repository is remotefilelog enabled or not""" | |
36 | return constants.SHALLOWREPO_REQUIREMENT in repo.requirements |
|
36 | return constants.SHALLOWREPO_REQUIREMENT in repo.requirements | |
37 |
|
37 | |||
38 | def getcachekey(reponame, file, id): |
|
38 | def getcachekey(reponame, file, id): | |
39 | pathhash = node.hex(hashlib.sha1(file).digest()) |
|
39 | pathhash = node.hex(hashlib.sha1(file).digest()) | |
40 | return os.path.join(reponame, pathhash[:2], pathhash[2:], id) |
|
40 | return os.path.join(reponame, pathhash[:2], pathhash[2:], id) | |
41 |
|
41 | |||
42 | def getlocalkey(file, id): |
|
42 | def getlocalkey(file, id): | |
43 | pathhash = node.hex(hashlib.sha1(file).digest()) |
|
43 | pathhash = node.hex(hashlib.sha1(file).digest()) | |
44 | return os.path.join(pathhash, id) |
|
44 | return os.path.join(pathhash, id) | |
45 |
|
45 | |||
46 | def getcachepath(ui, allowempty=False): |
|
46 | def getcachepath(ui, allowempty=False): | |
47 | cachepath = ui.config("remotefilelog", "cachepath") |
|
47 | cachepath = ui.config("remotefilelog", "cachepath") | |
48 | if not cachepath: |
|
48 | if not cachepath: | |
49 | if allowempty: |
|
49 | if allowempty: | |
50 | return None |
|
50 | return None | |
51 | else: |
|
51 | else: | |
52 | raise error.Abort(_("could not find config option " |
|
52 | raise error.Abort(_("could not find config option " | |
53 | "remotefilelog.cachepath")) |
|
53 | "remotefilelog.cachepath")) | |
54 | return util.expandpath(cachepath) |
|
54 | return util.expandpath(cachepath) | |
55 |
|
55 | |||
56 | def getcachepackpath(repo, category): |
|
56 | def getcachepackpath(repo, category): | |
57 | cachepath = getcachepath(repo.ui) |
|
57 | cachepath = getcachepath(repo.ui) | |
58 | if category != constants.FILEPACK_CATEGORY: |
|
58 | if category != constants.FILEPACK_CATEGORY: | |
59 | return os.path.join(cachepath, repo.name, 'packs', category) |
|
59 | return os.path.join(cachepath, repo.name, 'packs', category) | |
60 | else: |
|
60 | else: | |
61 | return os.path.join(cachepath, repo.name, 'packs') |
|
61 | return os.path.join(cachepath, repo.name, 'packs') | |
62 |
|
62 | |||
63 | def getlocalpackpath(base, category): |
|
63 | def getlocalpackpath(base, category): | |
64 | return os.path.join(base, 'packs', category) |
|
64 | return os.path.join(base, 'packs', category) | |
65 |
|
65 | |||
66 | def createrevlogtext(text, copyfrom=None, copyrev=None): |
|
66 | def createrevlogtext(text, copyfrom=None, copyrev=None): | |
67 | """returns a string that matches the revlog contents in a |
|
67 | """returns a string that matches the revlog contents in a | |
68 | traditional revlog |
|
68 | traditional revlog | |
69 | """ |
|
69 | """ | |
70 | meta = {} |
|
70 | meta = {} | |
71 | if copyfrom or text.startswith('\1\n'): |
|
71 | if copyfrom or text.startswith('\1\n'): | |
72 | if copyfrom: |
|
72 | if copyfrom: | |
73 | meta['copy'] = copyfrom |
|
73 | meta['copy'] = copyfrom | |
74 | meta['copyrev'] = copyrev |
|
74 | meta['copyrev'] = copyrev | |
75 | text = storageutil.packmeta(meta, text) |
|
75 | text = storageutil.packmeta(meta, text) | |
76 |
|
76 | |||
77 | return text |
|
77 | return text | |
78 |
|
78 | |||
79 | def parsemeta(text): |
|
79 | def parsemeta(text): | |
80 | """parse mercurial filelog metadata""" |
|
80 | """parse mercurial filelog metadata""" | |
81 | meta, size = storageutil.parsemeta(text) |
|
81 | meta, size = storageutil.parsemeta(text) | |
82 | if text.startswith('\1\n'): |
|
82 | if text.startswith('\1\n'): | |
83 | s = text.index('\1\n', 2) |
|
83 | s = text.index('\1\n', 2) | |
84 | text = text[s + 2:] |
|
84 | text = text[s + 2:] | |
85 | return meta or {}, text |
|
85 | return meta or {}, text | |
86 |
|
86 | |||
87 | def sumdicts(*dicts): |
|
87 | def sumdicts(*dicts): | |
88 | """Adds all the values of *dicts together into one dictionary. This assumes |
|
88 | """Adds all the values of *dicts together into one dictionary. This assumes | |
89 | the values in *dicts are all summable. |
|
89 | the values in *dicts are all summable. | |
90 |
|
90 | |||
91 | e.g. [{'a': 4', 'b': 2}, {'b': 3, 'c': 1}] -> {'a': 4, 'b': 5, 'c': 1} |
|
91 | e.g. [{'a': 4', 'b': 2}, {'b': 3, 'c': 1}] -> {'a': 4, 'b': 5, 'c': 1} | |
92 | """ |
|
92 | """ | |
93 | result = collections.defaultdict(lambda: 0) |
|
93 | result = collections.defaultdict(lambda: 0) | |
94 | for dict in dicts: |
|
94 | for dict in dicts: | |
95 | for k, v in dict.iteritems(): |
|
95 | for k, v in dict.iteritems(): | |
96 | result[k] += v |
|
96 | result[k] += v | |
97 | return result |
|
97 | return result | |
98 |
|
98 | |||
99 | def prefixkeys(dict, prefix): |
|
99 | def prefixkeys(dict, prefix): | |
100 | """Returns ``dict`` with ``prefix`` prepended to all its keys.""" |
|
100 | """Returns ``dict`` with ``prefix`` prepended to all its keys.""" | |
101 | result = {} |
|
101 | result = {} | |
102 | for k, v in dict.iteritems(): |
|
102 | for k, v in dict.iteritems(): | |
103 | result[prefix + k] = v |
|
103 | result[prefix + k] = v | |
104 | return result |
|
104 | return result | |
105 |
|
105 | |||
106 | def reportpackmetrics(ui, prefix, *stores): |
|
106 | def reportpackmetrics(ui, prefix, *stores): | |
107 | dicts = [s.getmetrics() for s in stores] |
|
107 | dicts = [s.getmetrics() for s in stores] | |
108 | dict = prefixkeys(sumdicts(*dicts), prefix + '_') |
|
108 | dict = prefixkeys(sumdicts(*dicts), prefix + '_') | |
109 | ui.log(prefix + "_packsizes", "", **pycompat.strkwargs(dict)) |
|
109 | ui.log(prefix + "_packsizes", "\n", **pycompat.strkwargs(dict)) | |
110 |
|
110 | |||
111 | def _parsepackmeta(metabuf): |
|
111 | def _parsepackmeta(metabuf): | |
112 | """parse datapack meta, bytes (<metadata-list>) -> dict |
|
112 | """parse datapack meta, bytes (<metadata-list>) -> dict | |
113 |
|
113 | |||
114 | The dict contains raw content - both keys and values are strings. |
|
114 | The dict contains raw content - both keys and values are strings. | |
115 | Upper-level business may want to convert some of them to other types like |
|
115 | Upper-level business may want to convert some of them to other types like | |
116 | integers, on their own. |
|
116 | integers, on their own. | |
117 |
|
117 | |||
118 | raise ValueError if the data is corrupted |
|
118 | raise ValueError if the data is corrupted | |
119 | """ |
|
119 | """ | |
120 | metadict = {} |
|
120 | metadict = {} | |
121 | offset = 0 |
|
121 | offset = 0 | |
122 | buflen = len(metabuf) |
|
122 | buflen = len(metabuf) | |
123 | while buflen - offset >= 3: |
|
123 | while buflen - offset >= 3: | |
124 | key = metabuf[offset] |
|
124 | key = metabuf[offset] | |
125 | offset += 1 |
|
125 | offset += 1 | |
126 | metalen = struct.unpack_from('!H', metabuf, offset)[0] |
|
126 | metalen = struct.unpack_from('!H', metabuf, offset)[0] | |
127 | offset += 2 |
|
127 | offset += 2 | |
128 | if offset + metalen > buflen: |
|
128 | if offset + metalen > buflen: | |
129 | raise ValueError('corrupted metadata: incomplete buffer') |
|
129 | raise ValueError('corrupted metadata: incomplete buffer') | |
130 | value = metabuf[offset:offset + metalen] |
|
130 | value = metabuf[offset:offset + metalen] | |
131 | metadict[key] = value |
|
131 | metadict[key] = value | |
132 | offset += metalen |
|
132 | offset += metalen | |
133 | if offset != buflen: |
|
133 | if offset != buflen: | |
134 | raise ValueError('corrupted metadata: redundant data') |
|
134 | raise ValueError('corrupted metadata: redundant data') | |
135 | return metadict |
|
135 | return metadict | |
136 |
|
136 | |||
137 | def _buildpackmeta(metadict): |
|
137 | def _buildpackmeta(metadict): | |
138 | """reverse of _parsepackmeta, dict -> bytes (<metadata-list>) |
|
138 | """reverse of _parsepackmeta, dict -> bytes (<metadata-list>) | |
139 |
|
139 | |||
140 | The dict contains raw content - both keys and values are strings. |
|
140 | The dict contains raw content - both keys and values are strings. | |
141 | Upper-level business may want to serialize some of other types (like |
|
141 | Upper-level business may want to serialize some of other types (like | |
142 | integers) to strings before calling this function. |
|
142 | integers) to strings before calling this function. | |
143 |
|
143 | |||
144 | raise ProgrammingError when metadata key is illegal, or ValueError if |
|
144 | raise ProgrammingError when metadata key is illegal, or ValueError if | |
145 | length limit is exceeded |
|
145 | length limit is exceeded | |
146 | """ |
|
146 | """ | |
147 | metabuf = '' |
|
147 | metabuf = '' | |
148 | for k, v in sorted((metadict or {}).iteritems()): |
|
148 | for k, v in sorted((metadict or {}).iteritems()): | |
149 | if len(k) != 1: |
|
149 | if len(k) != 1: | |
150 | raise error.ProgrammingError('packmeta: illegal key: %s' % k) |
|
150 | raise error.ProgrammingError('packmeta: illegal key: %s' % k) | |
151 | if len(v) > 0xfffe: |
|
151 | if len(v) > 0xfffe: | |
152 | raise ValueError('metadata value is too long: 0x%x > 0xfffe' |
|
152 | raise ValueError('metadata value is too long: 0x%x > 0xfffe' | |
153 | % len(v)) |
|
153 | % len(v)) | |
154 | metabuf += k |
|
154 | metabuf += k | |
155 | metabuf += struct.pack('!H', len(v)) |
|
155 | metabuf += struct.pack('!H', len(v)) | |
156 | metabuf += v |
|
156 | metabuf += v | |
157 | # len(metabuf) is guaranteed representable in 4 bytes, because there are |
|
157 | # len(metabuf) is guaranteed representable in 4 bytes, because there are | |
158 | # only 256 keys, and for each value, len(value) <= 0xfffe. |
|
158 | # only 256 keys, and for each value, len(value) <= 0xfffe. | |
159 | return metabuf |
|
159 | return metabuf | |
160 |
|
160 | |||
161 | _metaitemtypes = { |
|
161 | _metaitemtypes = { | |
162 | constants.METAKEYFLAG: (int, pycompat.long), |
|
162 | constants.METAKEYFLAG: (int, pycompat.long), | |
163 | constants.METAKEYSIZE: (int, pycompat.long), |
|
163 | constants.METAKEYSIZE: (int, pycompat.long), | |
164 | } |
|
164 | } | |
165 |
|
165 | |||
166 | def buildpackmeta(metadict): |
|
166 | def buildpackmeta(metadict): | |
167 | """like _buildpackmeta, but typechecks metadict and normalize it. |
|
167 | """like _buildpackmeta, but typechecks metadict and normalize it. | |
168 |
|
168 | |||
169 | This means, METAKEYSIZE and METAKEYSIZE should have integers as values, |
|
169 | This means, METAKEYSIZE and METAKEYSIZE should have integers as values, | |
170 | and METAKEYFLAG will be dropped if its value is 0. |
|
170 | and METAKEYFLAG will be dropped if its value is 0. | |
171 | """ |
|
171 | """ | |
172 | newmeta = {} |
|
172 | newmeta = {} | |
173 | for k, v in (metadict or {}).iteritems(): |
|
173 | for k, v in (metadict or {}).iteritems(): | |
174 | expectedtype = _metaitemtypes.get(k, (bytes,)) |
|
174 | expectedtype = _metaitemtypes.get(k, (bytes,)) | |
175 | if not isinstance(v, expectedtype): |
|
175 | if not isinstance(v, expectedtype): | |
176 | raise error.ProgrammingError('packmeta: wrong type of key %s' % k) |
|
176 | raise error.ProgrammingError('packmeta: wrong type of key %s' % k) | |
177 | # normalize int to binary buffer |
|
177 | # normalize int to binary buffer | |
178 | if int in expectedtype: |
|
178 | if int in expectedtype: | |
179 | # optimization: remove flag if it's 0 to save space |
|
179 | # optimization: remove flag if it's 0 to save space | |
180 | if k == constants.METAKEYFLAG and v == 0: |
|
180 | if k == constants.METAKEYFLAG and v == 0: | |
181 | continue |
|
181 | continue | |
182 | v = int2bin(v) |
|
182 | v = int2bin(v) | |
183 | newmeta[k] = v |
|
183 | newmeta[k] = v | |
184 | return _buildpackmeta(newmeta) |
|
184 | return _buildpackmeta(newmeta) | |
185 |
|
185 | |||
186 | def parsepackmeta(metabuf): |
|
186 | def parsepackmeta(metabuf): | |
187 | """like _parsepackmeta, but convert fields to desired types automatically. |
|
187 | """like _parsepackmeta, but convert fields to desired types automatically. | |
188 |
|
188 | |||
189 | This means, METAKEYFLAG and METAKEYSIZE fields will be converted to |
|
189 | This means, METAKEYFLAG and METAKEYSIZE fields will be converted to | |
190 | integers. |
|
190 | integers. | |
191 | """ |
|
191 | """ | |
192 | metadict = _parsepackmeta(metabuf) |
|
192 | metadict = _parsepackmeta(metabuf) | |
193 | for k, v in metadict.iteritems(): |
|
193 | for k, v in metadict.iteritems(): | |
194 | if k in _metaitemtypes and int in _metaitemtypes[k]: |
|
194 | if k in _metaitemtypes and int in _metaitemtypes[k]: | |
195 | metadict[k] = bin2int(v) |
|
195 | metadict[k] = bin2int(v) | |
196 | return metadict |
|
196 | return metadict | |
197 |
|
197 | |||
198 | def int2bin(n): |
|
198 | def int2bin(n): | |
199 | """convert a non-negative integer to raw binary buffer""" |
|
199 | """convert a non-negative integer to raw binary buffer""" | |
200 | buf = bytearray() |
|
200 | buf = bytearray() | |
201 | while n > 0: |
|
201 | while n > 0: | |
202 | buf.insert(0, n & 0xff) |
|
202 | buf.insert(0, n & 0xff) | |
203 | n >>= 8 |
|
203 | n >>= 8 | |
204 | return bytes(buf) |
|
204 | return bytes(buf) | |
205 |
|
205 | |||
206 | def bin2int(buf): |
|
206 | def bin2int(buf): | |
207 | """the reverse of int2bin, convert a binary buffer to an integer""" |
|
207 | """the reverse of int2bin, convert a binary buffer to an integer""" | |
208 | x = 0 |
|
208 | x = 0 | |
209 | for b in bytearray(buf): |
|
209 | for b in bytearray(buf): | |
210 | x <<= 8 |
|
210 | x <<= 8 | |
211 | x |= b |
|
211 | x |= b | |
212 | return x |
|
212 | return x | |
213 |
|
213 | |||
214 | def parsesizeflags(raw): |
|
214 | def parsesizeflags(raw): | |
215 | """given a remotefilelog blob, return (headersize, rawtextsize, flags) |
|
215 | """given a remotefilelog blob, return (headersize, rawtextsize, flags) | |
216 |
|
216 | |||
217 | see remotefilelogserver.createfileblob for the format. |
|
217 | see remotefilelogserver.createfileblob for the format. | |
218 | raise RuntimeError if the content is illformed. |
|
218 | raise RuntimeError if the content is illformed. | |
219 | """ |
|
219 | """ | |
220 | flags = revlog.REVIDX_DEFAULT_FLAGS |
|
220 | flags = revlog.REVIDX_DEFAULT_FLAGS | |
221 | size = None |
|
221 | size = None | |
222 | try: |
|
222 | try: | |
223 | index = raw.index('\0') |
|
223 | index = raw.index('\0') | |
224 | header = raw[:index] |
|
224 | header = raw[:index] | |
225 | if header.startswith('v'): |
|
225 | if header.startswith('v'): | |
226 | # v1 and above, header starts with 'v' |
|
226 | # v1 and above, header starts with 'v' | |
227 | if header.startswith('v1\n'): |
|
227 | if header.startswith('v1\n'): | |
228 | for s in header.split('\n'): |
|
228 | for s in header.split('\n'): | |
229 | if s.startswith(constants.METAKEYSIZE): |
|
229 | if s.startswith(constants.METAKEYSIZE): | |
230 | size = int(s[len(constants.METAKEYSIZE):]) |
|
230 | size = int(s[len(constants.METAKEYSIZE):]) | |
231 | elif s.startswith(constants.METAKEYFLAG): |
|
231 | elif s.startswith(constants.METAKEYFLAG): | |
232 | flags = int(s[len(constants.METAKEYFLAG):]) |
|
232 | flags = int(s[len(constants.METAKEYFLAG):]) | |
233 | else: |
|
233 | else: | |
234 | raise RuntimeError('unsupported remotefilelog header: %s' |
|
234 | raise RuntimeError('unsupported remotefilelog header: %s' | |
235 | % header) |
|
235 | % header) | |
236 | else: |
|
236 | else: | |
237 | # v0, str(int(size)) is the header |
|
237 | # v0, str(int(size)) is the header | |
238 | size = int(header) |
|
238 | size = int(header) | |
239 | except ValueError: |
|
239 | except ValueError: | |
240 | raise RuntimeError("unexpected remotefilelog header: illegal format") |
|
240 | raise RuntimeError("unexpected remotefilelog header: illegal format") | |
241 | if size is None: |
|
241 | if size is None: | |
242 | raise RuntimeError("unexpected remotefilelog header: no size found") |
|
242 | raise RuntimeError("unexpected remotefilelog header: no size found") | |
243 | return index + 1, size, flags |
|
243 | return index + 1, size, flags | |
244 |
|
244 | |||
245 | def buildfileblobheader(size, flags, version=None): |
|
245 | def buildfileblobheader(size, flags, version=None): | |
246 | """return the header of a remotefilelog blob. |
|
246 | """return the header of a remotefilelog blob. | |
247 |
|
247 | |||
248 | see remotefilelogserver.createfileblob for the format. |
|
248 | see remotefilelogserver.createfileblob for the format. | |
249 | approximately the reverse of parsesizeflags. |
|
249 | approximately the reverse of parsesizeflags. | |
250 |
|
250 | |||
251 | version could be 0 or 1, or None (auto decide). |
|
251 | version could be 0 or 1, or None (auto decide). | |
252 | """ |
|
252 | """ | |
253 | # choose v0 if flags is empty, otherwise v1 |
|
253 | # choose v0 if flags is empty, otherwise v1 | |
254 | if version is None: |
|
254 | if version is None: | |
255 | version = int(bool(flags)) |
|
255 | version = int(bool(flags)) | |
256 | if version == 1: |
|
256 | if version == 1: | |
257 | header = ('v1\n%s%d\n%s%d' |
|
257 | header = ('v1\n%s%d\n%s%d' | |
258 | % (constants.METAKEYSIZE, size, |
|
258 | % (constants.METAKEYSIZE, size, | |
259 | constants.METAKEYFLAG, flags)) |
|
259 | constants.METAKEYFLAG, flags)) | |
260 | elif version == 0: |
|
260 | elif version == 0: | |
261 | if flags: |
|
261 | if flags: | |
262 | raise error.ProgrammingError('fileblob v0 does not support flag') |
|
262 | raise error.ProgrammingError('fileblob v0 does not support flag') | |
263 | header = '%d' % size |
|
263 | header = '%d' % size | |
264 | else: |
|
264 | else: | |
265 | raise error.ProgrammingError('unknown fileblob version %d' % version) |
|
265 | raise error.ProgrammingError('unknown fileblob version %d' % version) | |
266 | return header |
|
266 | return header | |
267 |
|
267 | |||
268 | def ancestormap(raw): |
|
268 | def ancestormap(raw): | |
269 | offset, size, flags = parsesizeflags(raw) |
|
269 | offset, size, flags = parsesizeflags(raw) | |
270 | start = offset + size |
|
270 | start = offset + size | |
271 |
|
271 | |||
272 | mapping = {} |
|
272 | mapping = {} | |
273 | while start < len(raw): |
|
273 | while start < len(raw): | |
274 | divider = raw.index('\0', start + 80) |
|
274 | divider = raw.index('\0', start + 80) | |
275 |
|
275 | |||
276 | currentnode = raw[start:(start + 20)] |
|
276 | currentnode = raw[start:(start + 20)] | |
277 | p1 = raw[(start + 20):(start + 40)] |
|
277 | p1 = raw[(start + 20):(start + 40)] | |
278 | p2 = raw[(start + 40):(start + 60)] |
|
278 | p2 = raw[(start + 40):(start + 60)] | |
279 | linknode = raw[(start + 60):(start + 80)] |
|
279 | linknode = raw[(start + 60):(start + 80)] | |
280 | copyfrom = raw[(start + 80):divider] |
|
280 | copyfrom = raw[(start + 80):divider] | |
281 |
|
281 | |||
282 | mapping[currentnode] = (p1, p2, linknode, copyfrom) |
|
282 | mapping[currentnode] = (p1, p2, linknode, copyfrom) | |
283 | start = divider + 1 |
|
283 | start = divider + 1 | |
284 |
|
284 | |||
285 | return mapping |
|
285 | return mapping | |
286 |
|
286 | |||
287 | def readfile(path): |
|
287 | def readfile(path): | |
288 | f = open(path, 'rb') |
|
288 | f = open(path, 'rb') | |
289 | try: |
|
289 | try: | |
290 | result = f.read() |
|
290 | result = f.read() | |
291 |
|
291 | |||
292 | # we should never have empty files |
|
292 | # we should never have empty files | |
293 | if not result: |
|
293 | if not result: | |
294 | os.remove(path) |
|
294 | os.remove(path) | |
295 | raise IOError("empty file: %s" % path) |
|
295 | raise IOError("empty file: %s" % path) | |
296 |
|
296 | |||
297 | return result |
|
297 | return result | |
298 | finally: |
|
298 | finally: | |
299 | f.close() |
|
299 | f.close() | |
300 |
|
300 | |||
301 | def unlinkfile(filepath): |
|
301 | def unlinkfile(filepath): | |
302 | if pycompat.iswindows: |
|
302 | if pycompat.iswindows: | |
303 | # On Windows, os.unlink cannnot delete readonly files |
|
303 | # On Windows, os.unlink cannnot delete readonly files | |
304 | os.chmod(filepath, stat.S_IWUSR) |
|
304 | os.chmod(filepath, stat.S_IWUSR) | |
305 | os.unlink(filepath) |
|
305 | os.unlink(filepath) | |
306 |
|
306 | |||
307 | def renamefile(source, destination): |
|
307 | def renamefile(source, destination): | |
308 | if pycompat.iswindows: |
|
308 | if pycompat.iswindows: | |
309 | # On Windows, os.rename cannot rename readonly files |
|
309 | # On Windows, os.rename cannot rename readonly files | |
310 | # and cannot overwrite destination if it exists |
|
310 | # and cannot overwrite destination if it exists | |
311 | os.chmod(source, stat.S_IWUSR) |
|
311 | os.chmod(source, stat.S_IWUSR) | |
312 | if os.path.isfile(destination): |
|
312 | if os.path.isfile(destination): | |
313 | os.chmod(destination, stat.S_IWUSR) |
|
313 | os.chmod(destination, stat.S_IWUSR) | |
314 | os.unlink(destination) |
|
314 | os.unlink(destination) | |
315 |
|
315 | |||
316 | os.rename(source, destination) |
|
316 | os.rename(source, destination) | |
317 |
|
317 | |||
318 | def writefile(path, content, readonly=False): |
|
318 | def writefile(path, content, readonly=False): | |
319 | dirname, filename = os.path.split(path) |
|
319 | dirname, filename = os.path.split(path) | |
320 | if not os.path.exists(dirname): |
|
320 | if not os.path.exists(dirname): | |
321 | try: |
|
321 | try: | |
322 | os.makedirs(dirname) |
|
322 | os.makedirs(dirname) | |
323 | except OSError as ex: |
|
323 | except OSError as ex: | |
324 | if ex.errno != errno.EEXIST: |
|
324 | if ex.errno != errno.EEXIST: | |
325 | raise |
|
325 | raise | |
326 |
|
326 | |||
327 | fd, temp = tempfile.mkstemp(prefix='.%s-' % filename, dir=dirname) |
|
327 | fd, temp = tempfile.mkstemp(prefix='.%s-' % filename, dir=dirname) | |
328 | os.close(fd) |
|
328 | os.close(fd) | |
329 |
|
329 | |||
330 | try: |
|
330 | try: | |
331 | f = util.posixfile(temp, 'wb') |
|
331 | f = util.posixfile(temp, 'wb') | |
332 | f.write(content) |
|
332 | f.write(content) | |
333 | f.close() |
|
333 | f.close() | |
334 |
|
334 | |||
335 | if readonly: |
|
335 | if readonly: | |
336 | mode = 0o444 |
|
336 | mode = 0o444 | |
337 | else: |
|
337 | else: | |
338 | # tempfiles are created with 0o600, so we need to manually set the |
|
338 | # tempfiles are created with 0o600, so we need to manually set the | |
339 | # mode. |
|
339 | # mode. | |
340 | oldumask = os.umask(0) |
|
340 | oldumask = os.umask(0) | |
341 | # there's no way to get the umask without modifying it, so set it |
|
341 | # there's no way to get the umask without modifying it, so set it | |
342 | # back |
|
342 | # back | |
343 | os.umask(oldumask) |
|
343 | os.umask(oldumask) | |
344 | mode = ~oldumask |
|
344 | mode = ~oldumask | |
345 |
|
345 | |||
346 | renamefile(temp, path) |
|
346 | renamefile(temp, path) | |
347 | os.chmod(path, mode) |
|
347 | os.chmod(path, mode) | |
348 | except Exception: |
|
348 | except Exception: | |
349 | try: |
|
349 | try: | |
350 | unlinkfile(temp) |
|
350 | unlinkfile(temp) | |
351 | except OSError: |
|
351 | except OSError: | |
352 | pass |
|
352 | pass | |
353 | raise |
|
353 | raise | |
354 |
|
354 | |||
355 | def sortnodes(nodes, parentfunc): |
|
355 | def sortnodes(nodes, parentfunc): | |
356 | """Topologically sorts the nodes, using the parentfunc to find |
|
356 | """Topologically sorts the nodes, using the parentfunc to find | |
357 | the parents of nodes.""" |
|
357 | the parents of nodes.""" | |
358 | nodes = set(nodes) |
|
358 | nodes = set(nodes) | |
359 | childmap = {} |
|
359 | childmap = {} | |
360 | parentmap = {} |
|
360 | parentmap = {} | |
361 | roots = [] |
|
361 | roots = [] | |
362 |
|
362 | |||
363 | # Build a child and parent map |
|
363 | # Build a child and parent map | |
364 | for n in nodes: |
|
364 | for n in nodes: | |
365 | parents = [p for p in parentfunc(n) if p in nodes] |
|
365 | parents = [p for p in parentfunc(n) if p in nodes] | |
366 | parentmap[n] = set(parents) |
|
366 | parentmap[n] = set(parents) | |
367 | for p in parents: |
|
367 | for p in parents: | |
368 | childmap.setdefault(p, set()).add(n) |
|
368 | childmap.setdefault(p, set()).add(n) | |
369 | if not parents: |
|
369 | if not parents: | |
370 | roots.append(n) |
|
370 | roots.append(n) | |
371 |
|
371 | |||
372 | roots.sort() |
|
372 | roots.sort() | |
373 | # Process roots, adding children to the queue as they become roots |
|
373 | # Process roots, adding children to the queue as they become roots | |
374 | results = [] |
|
374 | results = [] | |
375 | while roots: |
|
375 | while roots: | |
376 | n = roots.pop(0) |
|
376 | n = roots.pop(0) | |
377 | results.append(n) |
|
377 | results.append(n) | |
378 | if n in childmap: |
|
378 | if n in childmap: | |
379 | children = childmap[n] |
|
379 | children = childmap[n] | |
380 | for c in children: |
|
380 | for c in children: | |
381 | childparents = parentmap[c] |
|
381 | childparents = parentmap[c] | |
382 | childparents.remove(n) |
|
382 | childparents.remove(n) | |
383 | if len(childparents) == 0: |
|
383 | if len(childparents) == 0: | |
384 | # insert at the beginning, that way child nodes |
|
384 | # insert at the beginning, that way child nodes | |
385 | # are likely to be output immediately after their |
|
385 | # are likely to be output immediately after their | |
386 | # parents. This gives better compression results. |
|
386 | # parents. This gives better compression results. | |
387 | roots.insert(0, c) |
|
387 | roots.insert(0, c) | |
388 |
|
388 | |||
389 | return results |
|
389 | return results | |
390 |
|
390 | |||
391 | def readexactly(stream, n): |
|
391 | def readexactly(stream, n): | |
392 | '''read n bytes from stream.read and abort if less was available''' |
|
392 | '''read n bytes from stream.read and abort if less was available''' | |
393 | s = stream.read(n) |
|
393 | s = stream.read(n) | |
394 | if len(s) < n: |
|
394 | if len(s) < n: | |
395 | raise error.Abort(_("stream ended unexpectedly" |
|
395 | raise error.Abort(_("stream ended unexpectedly" | |
396 | " (got %d bytes, expected %d)") |
|
396 | " (got %d bytes, expected %d)") | |
397 | % (len(s), n)) |
|
397 | % (len(s), n)) | |
398 | return s |
|
398 | return s | |
399 |
|
399 | |||
400 | def readunpack(stream, fmt): |
|
400 | def readunpack(stream, fmt): | |
401 | data = readexactly(stream, struct.calcsize(fmt)) |
|
401 | data = readexactly(stream, struct.calcsize(fmt)) | |
402 | return struct.unpack(fmt, data) |
|
402 | return struct.unpack(fmt, data) | |
403 |
|
403 | |||
404 | def readpath(stream): |
|
404 | def readpath(stream): | |
405 | rawlen = readexactly(stream, constants.FILENAMESIZE) |
|
405 | rawlen = readexactly(stream, constants.FILENAMESIZE) | |
406 | pathlen = struct.unpack(constants.FILENAMESTRUCT, rawlen)[0] |
|
406 | pathlen = struct.unpack(constants.FILENAMESTRUCT, rawlen)[0] | |
407 | return readexactly(stream, pathlen) |
|
407 | return readexactly(stream, pathlen) | |
408 |
|
408 | |||
409 | def readnodelist(stream): |
|
409 | def readnodelist(stream): | |
410 | rawlen = readexactly(stream, constants.NODECOUNTSIZE) |
|
410 | rawlen = readexactly(stream, constants.NODECOUNTSIZE) | |
411 | nodecount = struct.unpack(constants.NODECOUNTSTRUCT, rawlen)[0] |
|
411 | nodecount = struct.unpack(constants.NODECOUNTSTRUCT, rawlen)[0] | |
412 | for i in pycompat.xrange(nodecount): |
|
412 | for i in pycompat.xrange(nodecount): | |
413 | yield readexactly(stream, constants.NODESIZE) |
|
413 | yield readexactly(stream, constants.NODESIZE) | |
414 |
|
414 | |||
415 | def readpathlist(stream): |
|
415 | def readpathlist(stream): | |
416 | rawlen = readexactly(stream, constants.PATHCOUNTSIZE) |
|
416 | rawlen = readexactly(stream, constants.PATHCOUNTSIZE) | |
417 | pathcount = struct.unpack(constants.PATHCOUNTSTRUCT, rawlen)[0] |
|
417 | pathcount = struct.unpack(constants.PATHCOUNTSTRUCT, rawlen)[0] | |
418 | for i in pycompat.xrange(pathcount): |
|
418 | for i in pycompat.xrange(pathcount): | |
419 | yield readpath(stream) |
|
419 | yield readpath(stream) | |
420 |
|
420 | |||
421 | def getgid(groupname): |
|
421 | def getgid(groupname): | |
422 | try: |
|
422 | try: | |
423 | gid = grp.getgrnam(groupname).gr_gid |
|
423 | gid = grp.getgrnam(groupname).gr_gid | |
424 | return gid |
|
424 | return gid | |
425 | except KeyError: |
|
425 | except KeyError: | |
426 | return None |
|
426 | return None | |
427 |
|
427 | |||
428 | def setstickygroupdir(path, gid, warn=None): |
|
428 | def setstickygroupdir(path, gid, warn=None): | |
429 | if gid is None: |
|
429 | if gid is None: | |
430 | return |
|
430 | return | |
431 | try: |
|
431 | try: | |
432 | os.chown(path, -1, gid) |
|
432 | os.chown(path, -1, gid) | |
433 | os.chmod(path, 0o2775) |
|
433 | os.chmod(path, 0o2775) | |
434 | except (IOError, OSError) as ex: |
|
434 | except (IOError, OSError) as ex: | |
435 | if warn: |
|
435 | if warn: | |
436 | warn(_('unable to chown/chmod on %s: %s\n') % (path, ex)) |
|
436 | warn(_('unable to chown/chmod on %s: %s\n') % (path, ex)) | |
437 |
|
437 | |||
438 | def mkstickygroupdir(ui, path): |
|
438 | def mkstickygroupdir(ui, path): | |
439 | """Creates the given directory (if it doesn't exist) and give it a |
|
439 | """Creates the given directory (if it doesn't exist) and give it a | |
440 | particular group with setgid enabled.""" |
|
440 | particular group with setgid enabled.""" | |
441 | gid = None |
|
441 | gid = None | |
442 | groupname = ui.config("remotefilelog", "cachegroup") |
|
442 | groupname = ui.config("remotefilelog", "cachegroup") | |
443 | if groupname: |
|
443 | if groupname: | |
444 | gid = getgid(groupname) |
|
444 | gid = getgid(groupname) | |
445 | if gid is None: |
|
445 | if gid is None: | |
446 | ui.warn(_('unable to resolve group name: %s\n') % groupname) |
|
446 | ui.warn(_('unable to resolve group name: %s\n') % groupname) | |
447 |
|
447 | |||
448 | # we use a single stat syscall to test the existence and mode / group bit |
|
448 | # we use a single stat syscall to test the existence and mode / group bit | |
449 | st = None |
|
449 | st = None | |
450 | try: |
|
450 | try: | |
451 | st = os.stat(path) |
|
451 | st = os.stat(path) | |
452 | except OSError: |
|
452 | except OSError: | |
453 | pass |
|
453 | pass | |
454 |
|
454 | |||
455 | if st: |
|
455 | if st: | |
456 | # exists |
|
456 | # exists | |
457 | if (st.st_mode & 0o2775) != 0o2775 or st.st_gid != gid: |
|
457 | if (st.st_mode & 0o2775) != 0o2775 or st.st_gid != gid: | |
458 | # permission needs to be fixed |
|
458 | # permission needs to be fixed | |
459 | setstickygroupdir(path, gid, ui.warn) |
|
459 | setstickygroupdir(path, gid, ui.warn) | |
460 | return |
|
460 | return | |
461 |
|
461 | |||
462 | oldumask = os.umask(0o002) |
|
462 | oldumask = os.umask(0o002) | |
463 | try: |
|
463 | try: | |
464 | missingdirs = [path] |
|
464 | missingdirs = [path] | |
465 | path = os.path.dirname(path) |
|
465 | path = os.path.dirname(path) | |
466 | while path and not os.path.exists(path): |
|
466 | while path and not os.path.exists(path): | |
467 | missingdirs.append(path) |
|
467 | missingdirs.append(path) | |
468 | path = os.path.dirname(path) |
|
468 | path = os.path.dirname(path) | |
469 |
|
469 | |||
470 | for path in reversed(missingdirs): |
|
470 | for path in reversed(missingdirs): | |
471 | try: |
|
471 | try: | |
472 | os.mkdir(path) |
|
472 | os.mkdir(path) | |
473 | except OSError as ex: |
|
473 | except OSError as ex: | |
474 | if ex.errno != errno.EEXIST: |
|
474 | if ex.errno != errno.EEXIST: | |
475 | raise |
|
475 | raise | |
476 |
|
476 | |||
477 | for path in missingdirs: |
|
477 | for path in missingdirs: | |
478 | setstickygroupdir(path, gid, ui.warn) |
|
478 | setstickygroupdir(path, gid, ui.warn) | |
479 | finally: |
|
479 | finally: | |
480 | os.umask(oldumask) |
|
480 | os.umask(oldumask) | |
481 |
|
481 | |||
482 | def getusername(ui): |
|
482 | def getusername(ui): | |
483 | try: |
|
483 | try: | |
484 | return stringutil.shortuser(ui.username()) |
|
484 | return stringutil.shortuser(ui.username()) | |
485 | except Exception: |
|
485 | except Exception: | |
486 | return 'unknown' |
|
486 | return 'unknown' | |
487 |
|
487 | |||
488 | def getreponame(ui): |
|
488 | def getreponame(ui): | |
489 | reponame = ui.config('paths', 'default') |
|
489 | reponame = ui.config('paths', 'default') | |
490 | if reponame: |
|
490 | if reponame: | |
491 | return os.path.basename(reponame) |
|
491 | return os.path.basename(reponame) | |
492 | return "unknown" |
|
492 | return "unknown" |
General Comments 0
You need to be logged in to leave comments.
Login now