##// END OF EJS Templates
largefiles: mark headre as bytes regex...
Augie Fackler -
r36330:3ac8b5c1 default
parent child Browse files
Show More
@@ -1,190 +1,190 b''
1 # Copyright 2011 Fog Creek Software
1 # Copyright 2011 Fog Creek Software
2 #
2 #
3 # This software may be used and distributed according to the terms of the
3 # This software may be used and distributed according to the terms of the
4 # GNU General Public License version 2 or any later version.
4 # GNU General Public License version 2 or any later version.
5 from __future__ import absolute_import
5 from __future__ import absolute_import
6
6
7 import os
7 import os
8 import re
8 import re
9
9
10 from mercurial.i18n import _
10 from mercurial.i18n import _
11
11
12 from mercurial import (
12 from mercurial import (
13 error,
13 error,
14 httppeer,
14 httppeer,
15 util,
15 util,
16 wireproto,
16 wireproto,
17 wireprototypes,
17 wireprototypes,
18 )
18 )
19
19
20 from . import (
20 from . import (
21 lfutil,
21 lfutil,
22 )
22 )
23
23
24 urlerr = util.urlerr
24 urlerr = util.urlerr
25 urlreq = util.urlreq
25 urlreq = util.urlreq
26
26
27 LARGEFILES_REQUIRED_MSG = ('\nThis repository uses the largefiles extension.'
27 LARGEFILES_REQUIRED_MSG = ('\nThis repository uses the largefiles extension.'
28 '\n\nPlease enable it in your Mercurial config '
28 '\n\nPlease enable it in your Mercurial config '
29 'file.\n')
29 'file.\n')
30
30
31 # these will all be replaced by largefiles.uisetup
31 # these will all be replaced by largefiles.uisetup
32 ssholdcallstream = None
32 ssholdcallstream = None
33 httpoldcallstream = None
33 httpoldcallstream = None
34
34
35 def putlfile(repo, proto, sha):
35 def putlfile(repo, proto, sha):
36 '''Server command for putting a largefile into a repository's local store
36 '''Server command for putting a largefile into a repository's local store
37 and into the user cache.'''
37 and into the user cache.'''
38 with proto.mayberedirectstdio() as output:
38 with proto.mayberedirectstdio() as output:
39 path = lfutil.storepath(repo, sha)
39 path = lfutil.storepath(repo, sha)
40 util.makedirs(os.path.dirname(path))
40 util.makedirs(os.path.dirname(path))
41 tmpfp = util.atomictempfile(path, createmode=repo.store.createmode)
41 tmpfp = util.atomictempfile(path, createmode=repo.store.createmode)
42
42
43 try:
43 try:
44 proto.forwardpayload(tmpfp)
44 proto.forwardpayload(tmpfp)
45 tmpfp._fp.seek(0)
45 tmpfp._fp.seek(0)
46 if sha != lfutil.hexsha1(tmpfp._fp):
46 if sha != lfutil.hexsha1(tmpfp._fp):
47 raise IOError(0, _('largefile contents do not match hash'))
47 raise IOError(0, _('largefile contents do not match hash'))
48 tmpfp.close()
48 tmpfp.close()
49 lfutil.linktousercache(repo, sha)
49 lfutil.linktousercache(repo, sha)
50 except IOError as e:
50 except IOError as e:
51 repo.ui.warn(_('largefiles: failed to put %s into store: %s\n') %
51 repo.ui.warn(_('largefiles: failed to put %s into store: %s\n') %
52 (sha, e.strerror))
52 (sha, e.strerror))
53 return wireproto.pushres(1, output.getvalue() if output else '')
53 return wireproto.pushres(1, output.getvalue() if output else '')
54 finally:
54 finally:
55 tmpfp.discard()
55 tmpfp.discard()
56
56
57 return wireproto.pushres(0, output.getvalue() if output else '')
57 return wireproto.pushres(0, output.getvalue() if output else '')
58
58
59 def getlfile(repo, proto, sha):
59 def getlfile(repo, proto, sha):
60 '''Server command for retrieving a largefile from the repository-local
60 '''Server command for retrieving a largefile from the repository-local
61 cache or user cache.'''
61 cache or user cache.'''
62 filename = lfutil.findfile(repo, sha)
62 filename = lfutil.findfile(repo, sha)
63 if not filename:
63 if not filename:
64 raise error.Abort(_('requested largefile %s not present in cache')
64 raise error.Abort(_('requested largefile %s not present in cache')
65 % sha)
65 % sha)
66 f = open(filename, 'rb')
66 f = open(filename, 'rb')
67 length = os.fstat(f.fileno())[6]
67 length = os.fstat(f.fileno())[6]
68
68
69 # Since we can't set an HTTP content-length header here, and
69 # Since we can't set an HTTP content-length header here, and
70 # Mercurial core provides no way to give the length of a streamres
70 # Mercurial core provides no way to give the length of a streamres
71 # (and reading the entire file into RAM would be ill-advised), we
71 # (and reading the entire file into RAM would be ill-advised), we
72 # just send the length on the first line of the response, like the
72 # just send the length on the first line of the response, like the
73 # ssh proto does for string responses.
73 # ssh proto does for string responses.
74 def generator():
74 def generator():
75 yield '%d\n' % length
75 yield '%d\n' % length
76 for chunk in util.filechunkiter(f):
76 for chunk in util.filechunkiter(f):
77 yield chunk
77 yield chunk
78 return wireproto.streamres_legacy(gen=generator())
78 return wireproto.streamres_legacy(gen=generator())
79
79
80 def statlfile(repo, proto, sha):
80 def statlfile(repo, proto, sha):
81 '''Server command for checking if a largefile is present - returns '2\n' if
81 '''Server command for checking if a largefile is present - returns '2\n' if
82 the largefile is missing, '0\n' if it seems to be in good condition.
82 the largefile is missing, '0\n' if it seems to be in good condition.
83
83
84 The value 1 is reserved for mismatched checksum, but that is too expensive
84 The value 1 is reserved for mismatched checksum, but that is too expensive
85 to be verified on every stat and must be caught be running 'hg verify'
85 to be verified on every stat and must be caught be running 'hg verify'
86 server side.'''
86 server side.'''
87 filename = lfutil.findfile(repo, sha)
87 filename = lfutil.findfile(repo, sha)
88 if not filename:
88 if not filename:
89 return wireprototypes.bytesresponse('2\n')
89 return wireprototypes.bytesresponse('2\n')
90 return wireprototypes.bytesresponse('0\n')
90 return wireprototypes.bytesresponse('0\n')
91
91
92 def wirereposetup(ui, repo):
92 def wirereposetup(ui, repo):
93 class lfileswirerepository(repo.__class__):
93 class lfileswirerepository(repo.__class__):
94 def putlfile(self, sha, fd):
94 def putlfile(self, sha, fd):
95 # unfortunately, httprepository._callpush tries to convert its
95 # unfortunately, httprepository._callpush tries to convert its
96 # input file-like into a bundle before sending it, so we can't use
96 # input file-like into a bundle before sending it, so we can't use
97 # it ...
97 # it ...
98 if issubclass(self.__class__, httppeer.httppeer):
98 if issubclass(self.__class__, httppeer.httppeer):
99 res = self._call('putlfile', data=fd, sha=sha,
99 res = self._call('putlfile', data=fd, sha=sha,
100 headers={'content-type':'application/mercurial-0.1'})
100 headers={'content-type':'application/mercurial-0.1'})
101 try:
101 try:
102 d, output = res.split('\n', 1)
102 d, output = res.split('\n', 1)
103 for l in output.splitlines(True):
103 for l in output.splitlines(True):
104 self.ui.warn(_('remote: '), l) # assume l ends with \n
104 self.ui.warn(_('remote: '), l) # assume l ends with \n
105 return int(d)
105 return int(d)
106 except ValueError:
106 except ValueError:
107 self.ui.warn(_('unexpected putlfile response: %r\n') % res)
107 self.ui.warn(_('unexpected putlfile response: %r\n') % res)
108 return 1
108 return 1
109 # ... but we can't use sshrepository._call because the data=
109 # ... but we can't use sshrepository._call because the data=
110 # argument won't get sent, and _callpush does exactly what we want
110 # argument won't get sent, and _callpush does exactly what we want
111 # in this case: send the data straight through
111 # in this case: send the data straight through
112 else:
112 else:
113 try:
113 try:
114 ret, output = self._callpush("putlfile", fd, sha=sha)
114 ret, output = self._callpush("putlfile", fd, sha=sha)
115 if ret == "":
115 if ret == "":
116 raise error.ResponseError(_('putlfile failed:'),
116 raise error.ResponseError(_('putlfile failed:'),
117 output)
117 output)
118 return int(ret)
118 return int(ret)
119 except IOError:
119 except IOError:
120 return 1
120 return 1
121 except ValueError:
121 except ValueError:
122 raise error.ResponseError(
122 raise error.ResponseError(
123 _('putlfile failed (unexpected response):'), ret)
123 _('putlfile failed (unexpected response):'), ret)
124
124
125 def getlfile(self, sha):
125 def getlfile(self, sha):
126 """returns an iterable with the chunks of the file with sha sha"""
126 """returns an iterable with the chunks of the file with sha sha"""
127 stream = self._callstream("getlfile", sha=sha)
127 stream = self._callstream("getlfile", sha=sha)
128 length = stream.readline()
128 length = stream.readline()
129 try:
129 try:
130 length = int(length)
130 length = int(length)
131 except ValueError:
131 except ValueError:
132 self._abort(error.ResponseError(_("unexpected response:"),
132 self._abort(error.ResponseError(_("unexpected response:"),
133 length))
133 length))
134
134
135 # SSH streams will block if reading more than length
135 # SSH streams will block if reading more than length
136 for chunk in util.filechunkiter(stream, limit=length):
136 for chunk in util.filechunkiter(stream, limit=length):
137 yield chunk
137 yield chunk
138 # HTTP streams must hit the end to process the last empty
138 # HTTP streams must hit the end to process the last empty
139 # chunk of Chunked-Encoding so the connection can be reused.
139 # chunk of Chunked-Encoding so the connection can be reused.
140 if issubclass(self.__class__, httppeer.httppeer):
140 if issubclass(self.__class__, httppeer.httppeer):
141 chunk = stream.read(1)
141 chunk = stream.read(1)
142 if chunk:
142 if chunk:
143 self._abort(error.ResponseError(_("unexpected response:"),
143 self._abort(error.ResponseError(_("unexpected response:"),
144 chunk))
144 chunk))
145
145
146 @wireproto.batchable
146 @wireproto.batchable
147 def statlfile(self, sha):
147 def statlfile(self, sha):
148 f = wireproto.future()
148 f = wireproto.future()
149 result = {'sha': sha}
149 result = {'sha': sha}
150 yield result, f
150 yield result, f
151 try:
151 try:
152 yield int(f.value)
152 yield int(f.value)
153 except (ValueError, urlerr.httperror):
153 except (ValueError, urlerr.httperror):
154 # If the server returns anything but an integer followed by a
154 # If the server returns anything but an integer followed by a
155 # newline, newline, it's not speaking our language; if we get
155 # newline, newline, it's not speaking our language; if we get
156 # an HTTP error, we can't be sure the largefile is present;
156 # an HTTP error, we can't be sure the largefile is present;
157 # either way, consider it missing.
157 # either way, consider it missing.
158 yield 2
158 yield 2
159
159
160 repo.__class__ = lfileswirerepository
160 repo.__class__ = lfileswirerepository
161
161
162 # advertise the largefiles=serve capability
162 # advertise the largefiles=serve capability
163 def _capabilities(orig, repo, proto):
163 def _capabilities(orig, repo, proto):
164 '''announce largefile server capability'''
164 '''announce largefile server capability'''
165 caps = orig(repo, proto)
165 caps = orig(repo, proto)
166 caps.append('largefiles=serve')
166 caps.append('largefiles=serve')
167 return caps
167 return caps
168
168
169 def heads(repo, proto):
169 def heads(repo, proto):
170 '''Wrap server command - largefile capable clients will know to call
170 '''Wrap server command - largefile capable clients will know to call
171 lheads instead'''
171 lheads instead'''
172 if lfutil.islfilesrepo(repo):
172 if lfutil.islfilesrepo(repo):
173 return wireproto.ooberror(LARGEFILES_REQUIRED_MSG)
173 return wireproto.ooberror(LARGEFILES_REQUIRED_MSG)
174 return wireproto.heads(repo, proto)
174 return wireproto.heads(repo, proto)
175
175
176 def sshrepocallstream(self, cmd, **args):
176 def sshrepocallstream(self, cmd, **args):
177 if cmd == 'heads' and self.capable('largefiles'):
177 if cmd == 'heads' and self.capable('largefiles'):
178 cmd = 'lheads'
178 cmd = 'lheads'
179 if cmd == 'batch' and self.capable('largefiles'):
179 if cmd == 'batch' and self.capable('largefiles'):
180 args[r'cmds'] = args[r'cmds'].replace('heads ', 'lheads ')
180 args[r'cmds'] = args[r'cmds'].replace('heads ', 'lheads ')
181 return ssholdcallstream(self, cmd, **args)
181 return ssholdcallstream(self, cmd, **args)
182
182
183 headsre = re.compile(r'(^|;)heads\b')
183 headsre = re.compile(br'(^|;)heads\b')
184
184
185 def httprepocallstream(self, cmd, **args):
185 def httprepocallstream(self, cmd, **args):
186 if cmd == 'heads' and self.capable('largefiles'):
186 if cmd == 'heads' and self.capable('largefiles'):
187 cmd = 'lheads'
187 cmd = 'lheads'
188 if cmd == 'batch' and self.capable('largefiles'):
188 if cmd == 'batch' and self.capable('largefiles'):
189 args[r'cmds'] = headsre.sub('lheads', args[r'cmds'])
189 args[r'cmds'] = headsre.sub('lheads', args[r'cmds'])
190 return httpoldcallstream(self, cmd, **args)
190 return httpoldcallstream(self, cmd, **args)
General Comments 0
You need to be logged in to leave comments. Login now