##// END OF EJS Templates
largefiles: make the protocol hack for replacing heads with lheads more precise...
Mads Kiilerich -
r19917:cff331cb default
parent child Browse files
Show More
@@ -1,174 +1,177 b''
1 # Copyright 2011 Fog Creek Software
1 # Copyright 2011 Fog Creek Software
2 #
2 #
3 # This software may be used and distributed according to the terms of the
3 # This software may be used and distributed according to the terms of the
4 # GNU General Public License version 2 or any later version.
4 # GNU General Public License version 2 or any later version.
5
5
6 import os
6 import os
7 import urllib2
7 import urllib2
8 import re
8
9
9 from mercurial import error, httppeer, util, wireproto
10 from mercurial import error, httppeer, util, wireproto
10 from mercurial.wireproto import batchable, future
11 from mercurial.wireproto import batchable, future
11 from mercurial.i18n import _
12 from mercurial.i18n import _
12
13
13 import lfutil
14 import lfutil
14
15
15 LARGEFILES_REQUIRED_MSG = ('\nThis repository uses the largefiles extension.'
16 LARGEFILES_REQUIRED_MSG = ('\nThis repository uses the largefiles extension.'
16 '\n\nPlease enable it in your Mercurial config '
17 '\n\nPlease enable it in your Mercurial config '
17 'file.\n')
18 'file.\n')
18
19
19 # these will all be replaced by largefiles.uisetup
20 # these will all be replaced by largefiles.uisetup
20 capabilitiesorig = None
21 capabilitiesorig = None
21 ssholdcallstream = None
22 ssholdcallstream = None
22 httpoldcallstream = None
23 httpoldcallstream = None
23
24
24 def putlfile(repo, proto, sha):
25 def putlfile(repo, proto, sha):
25 '''Put a largefile into a repository's local store and into the
26 '''Put a largefile into a repository's local store and into the
26 user cache.'''
27 user cache.'''
27 proto.redirect()
28 proto.redirect()
28
29
29 path = lfutil.storepath(repo, sha)
30 path = lfutil.storepath(repo, sha)
30 util.makedirs(os.path.dirname(path))
31 util.makedirs(os.path.dirname(path))
31 tmpfp = util.atomictempfile(path, createmode=repo.store.createmode)
32 tmpfp = util.atomictempfile(path, createmode=repo.store.createmode)
32
33
33 try:
34 try:
34 try:
35 try:
35 proto.getfile(tmpfp)
36 proto.getfile(tmpfp)
36 tmpfp._fp.seek(0)
37 tmpfp._fp.seek(0)
37 if sha != lfutil.hexsha1(tmpfp._fp):
38 if sha != lfutil.hexsha1(tmpfp._fp):
38 raise IOError(0, _('largefile contents do not match hash'))
39 raise IOError(0, _('largefile contents do not match hash'))
39 tmpfp.close()
40 tmpfp.close()
40 lfutil.linktousercache(repo, sha)
41 lfutil.linktousercache(repo, sha)
41 except IOError, e:
42 except IOError, e:
42 repo.ui.warn(_('largefiles: failed to put %s into store: %s') %
43 repo.ui.warn(_('largefiles: failed to put %s into store: %s') %
43 (sha, e.strerror))
44 (sha, e.strerror))
44 return wireproto.pushres(1)
45 return wireproto.pushres(1)
45 finally:
46 finally:
46 tmpfp.discard()
47 tmpfp.discard()
47
48
48 return wireproto.pushres(0)
49 return wireproto.pushres(0)
49
50
50 def getlfile(repo, proto, sha):
51 def getlfile(repo, proto, sha):
51 '''Retrieve a largefile from the repository-local cache or system
52 '''Retrieve a largefile from the repository-local cache or system
52 cache.'''
53 cache.'''
53 filename = lfutil.findfile(repo, sha)
54 filename = lfutil.findfile(repo, sha)
54 if not filename:
55 if not filename:
55 raise util.Abort(_('requested largefile %s not present in cache') % sha)
56 raise util.Abort(_('requested largefile %s not present in cache') % sha)
56 f = open(filename, 'rb')
57 f = open(filename, 'rb')
57 length = os.fstat(f.fileno())[6]
58 length = os.fstat(f.fileno())[6]
58
59
59 # Since we can't set an HTTP content-length header here, and
60 # Since we can't set an HTTP content-length header here, and
60 # Mercurial core provides no way to give the length of a streamres
61 # Mercurial core provides no way to give the length of a streamres
61 # (and reading the entire file into RAM would be ill-advised), we
62 # (and reading the entire file into RAM would be ill-advised), we
62 # just send the length on the first line of the response, like the
63 # just send the length on the first line of the response, like the
63 # ssh proto does for string responses.
64 # ssh proto does for string responses.
64 def generator():
65 def generator():
65 yield '%d\n' % length
66 yield '%d\n' % length
66 for chunk in util.filechunkiter(f):
67 for chunk in util.filechunkiter(f):
67 yield chunk
68 yield chunk
68 return wireproto.streamres(generator())
69 return wireproto.streamres(generator())
69
70
70 def statlfile(repo, proto, sha):
71 def statlfile(repo, proto, sha):
71 '''Return '2\n' if the largefile is missing, '0\n' if it seems to be in
72 '''Return '2\n' if the largefile is missing, '0\n' if it seems to be in
72 good condition.
73 good condition.
73
74
74 The value 1 is reserved for mismatched checksum, but that is too expensive
75 The value 1 is reserved for mismatched checksum, but that is too expensive
75 to be verified on every stat and must be caught be running 'hg verify'
76 to be verified on every stat and must be caught be running 'hg verify'
76 server side.'''
77 server side.'''
77 filename = lfutil.findfile(repo, sha)
78 filename = lfutil.findfile(repo, sha)
78 if not filename:
79 if not filename:
79 return '2\n'
80 return '2\n'
80 return '0\n'
81 return '0\n'
81
82
82 def wirereposetup(ui, repo):
83 def wirereposetup(ui, repo):
83 class lfileswirerepository(repo.__class__):
84 class lfileswirerepository(repo.__class__):
84 def putlfile(self, sha, fd):
85 def putlfile(self, sha, fd):
85 # unfortunately, httprepository._callpush tries to convert its
86 # unfortunately, httprepository._callpush tries to convert its
86 # input file-like into a bundle before sending it, so we can't use
87 # input file-like into a bundle before sending it, so we can't use
87 # it ...
88 # it ...
88 if issubclass(self.__class__, httppeer.httppeer):
89 if issubclass(self.__class__, httppeer.httppeer):
89 res = None
90 res = None
90 try:
91 try:
91 res = self._call('putlfile', data=fd, sha=sha,
92 res = self._call('putlfile', data=fd, sha=sha,
92 headers={'content-type':'application/mercurial-0.1'})
93 headers={'content-type':'application/mercurial-0.1'})
93 d, output = res.split('\n', 1)
94 d, output = res.split('\n', 1)
94 for l in output.splitlines(True):
95 for l in output.splitlines(True):
95 self.ui.warn(_('remote: '), l, '\n')
96 self.ui.warn(_('remote: '), l, '\n')
96 return int(d)
97 return int(d)
97 except (ValueError, urllib2.HTTPError):
98 except (ValueError, urllib2.HTTPError):
98 self.ui.warn(_('unexpected putlfile response: %s') % res)
99 self.ui.warn(_('unexpected putlfile response: %s') % res)
99 return 1
100 return 1
100 # ... but we can't use sshrepository._call because the data=
101 # ... but we can't use sshrepository._call because the data=
101 # argument won't get sent, and _callpush does exactly what we want
102 # argument won't get sent, and _callpush does exactly what we want
102 # in this case: send the data straight through
103 # in this case: send the data straight through
103 else:
104 else:
104 try:
105 try:
105 ret, output = self._callpush("putlfile", fd, sha=sha)
106 ret, output = self._callpush("putlfile", fd, sha=sha)
106 if ret == "":
107 if ret == "":
107 raise error.ResponseError(_('putlfile failed:'),
108 raise error.ResponseError(_('putlfile failed:'),
108 output)
109 output)
109 return int(ret)
110 return int(ret)
110 except IOError:
111 except IOError:
111 return 1
112 return 1
112 except ValueError:
113 except ValueError:
113 raise error.ResponseError(
114 raise error.ResponseError(
114 _('putlfile failed (unexpected response):'), ret)
115 _('putlfile failed (unexpected response):'), ret)
115
116
116 def getlfile(self, sha):
117 def getlfile(self, sha):
117 """returns an iterable with the chunks of the file with sha sha"""
118 """returns an iterable with the chunks of the file with sha sha"""
118 stream = self._callstream("getlfile", sha=sha)
119 stream = self._callstream("getlfile", sha=sha)
119 length = stream.readline()
120 length = stream.readline()
120 try:
121 try:
121 length = int(length)
122 length = int(length)
122 except ValueError:
123 except ValueError:
123 self._abort(error.ResponseError(_("unexpected response:"),
124 self._abort(error.ResponseError(_("unexpected response:"),
124 length))
125 length))
125
126
126 # SSH streams will block if reading more than length
127 # SSH streams will block if reading more than length
127 for chunk in util.filechunkiter(stream, 128 * 1024, length):
128 for chunk in util.filechunkiter(stream, 128 * 1024, length):
128 yield chunk
129 yield chunk
129 # HTTP streams must hit the end to process the last empty
130 # HTTP streams must hit the end to process the last empty
130 # chunk of Chunked-Encoding so the connection can be reused.
131 # chunk of Chunked-Encoding so the connection can be reused.
131 if issubclass(self.__class__, httppeer.httppeer):
132 if issubclass(self.__class__, httppeer.httppeer):
132 chunk = stream.read(1)
133 chunk = stream.read(1)
133 if chunk:
134 if chunk:
134 self._abort(error.ResponseError(_("unexpected response:"),
135 self._abort(error.ResponseError(_("unexpected response:"),
135 chunk))
136 chunk))
136
137
137 @batchable
138 @batchable
138 def statlfile(self, sha):
139 def statlfile(self, sha):
139 f = future()
140 f = future()
140 result = {'sha': sha}
141 result = {'sha': sha}
141 yield result, f
142 yield result, f
142 try:
143 try:
143 yield int(f.value)
144 yield int(f.value)
144 except (ValueError, urllib2.HTTPError):
145 except (ValueError, urllib2.HTTPError):
145 # If the server returns anything but an integer followed by a
146 # If the server returns anything but an integer followed by a
146 # newline, newline, it's not speaking our language; if we get
147 # newline, newline, it's not speaking our language; if we get
147 # an HTTP error, we can't be sure the largefile is present;
148 # an HTTP error, we can't be sure the largefile is present;
148 # either way, consider it missing.
149 # either way, consider it missing.
149 yield 2
150 yield 2
150
151
151 repo.__class__ = lfileswirerepository
152 repo.__class__ = lfileswirerepository
152
153
153 # advertise the largefiles=serve capability
154 # advertise the largefiles=serve capability
154 def capabilities(repo, proto):
155 def capabilities(repo, proto):
155 return capabilitiesorig(repo, proto) + ' largefiles=serve'
156 return capabilitiesorig(repo, proto) + ' largefiles=serve'
156
157
157 def heads(repo, proto):
158 def heads(repo, proto):
158 if lfutil.islfilesrepo(repo):
159 if lfutil.islfilesrepo(repo):
159 return wireproto.ooberror(LARGEFILES_REQUIRED_MSG)
160 return wireproto.ooberror(LARGEFILES_REQUIRED_MSG)
160 return wireproto.heads(repo, proto)
161 return wireproto.heads(repo, proto)
161
162
162 def sshrepocallstream(self, cmd, **args):
163 def sshrepocallstream(self, cmd, **args):
163 if cmd == 'heads' and self.capable('largefiles'):
164 if cmd == 'heads' and self.capable('largefiles'):
164 cmd = 'lheads'
165 cmd = 'lheads'
165 if cmd == 'batch' and self.capable('largefiles'):
166 if cmd == 'batch' and self.capable('largefiles'):
166 args['cmds'] = args['cmds'].replace('heads ', 'lheads ')
167 args['cmds'] = args['cmds'].replace('heads ', 'lheads ')
167 return ssholdcallstream(self, cmd, **args)
168 return ssholdcallstream(self, cmd, **args)
168
169
170 headsre = re.compile(r'(^|;)heads\b')
171
169 def httprepocallstream(self, cmd, **args):
172 def httprepocallstream(self, cmd, **args):
170 if cmd == 'heads' and self.capable('largefiles'):
173 if cmd == 'heads' and self.capable('largefiles'):
171 cmd = 'lheads'
174 cmd = 'lheads'
172 if cmd == 'batch' and self.capable('largefiles'):
175 if cmd == 'batch' and self.capable('largefiles'):
173 args['cmds'] = args['cmds'].replace('heads ', 'lheads ')
176 args['cmds'] = headsre.sub('lheads', args['cmds'])
174 return httpoldcallstream(self, cmd, **args)
177 return httpoldcallstream(self, cmd, **args)
General Comments 0
You need to be logged in to leave comments. Login now