Show More
@@ -1,640 +1,640 b'' | |||||
1 | # blobstore.py - local and remote (speaking Git-LFS protocol) blob storages |
|
1 | # blobstore.py - local and remote (speaking Git-LFS protocol) blob storages | |
2 | # |
|
2 | # | |
3 | # Copyright 2017 Facebook, Inc. |
|
3 | # Copyright 2017 Facebook, Inc. | |
4 | # |
|
4 | # | |
5 | # This software may be used and distributed according to the terms of the |
|
5 | # This software may be used and distributed according to the terms of the | |
6 | # GNU General Public License version 2 or any later version. |
|
6 | # GNU General Public License version 2 or any later version. | |
7 |
|
7 | |||
8 | from __future__ import absolute_import |
|
8 | from __future__ import absolute_import | |
9 |
|
9 | |||
10 | import errno |
|
10 | import errno | |
11 | import hashlib |
|
11 | import hashlib | |
12 | import json |
|
12 | import json | |
13 | import os |
|
13 | import os | |
14 | import re |
|
14 | import re | |
15 | import socket |
|
15 | import socket | |
16 |
|
16 | |||
17 | from mercurial.i18n import _ |
|
17 | from mercurial.i18n import _ | |
18 |
|
18 | |||
19 | from mercurial import ( |
|
19 | from mercurial import ( | |
20 | encoding, |
|
20 | encoding, | |
21 | error, |
|
21 | error, | |
22 | pathutil, |
|
22 | pathutil, | |
23 | pycompat, |
|
23 | pycompat, | |
24 | url as urlmod, |
|
24 | url as urlmod, | |
25 | util, |
|
25 | util, | |
26 | vfs as vfsmod, |
|
26 | vfs as vfsmod, | |
27 | worker, |
|
27 | worker, | |
28 | ) |
|
28 | ) | |
29 |
|
29 | |||
30 | from mercurial.utils import ( |
|
30 | from mercurial.utils import ( | |
31 | stringutil, |
|
31 | stringutil, | |
32 | ) |
|
32 | ) | |
33 |
|
33 | |||
34 | from ..largefiles import lfutil |
|
34 | from ..largefiles import lfutil | |
35 |
|
35 | |||
36 | # 64 bytes for SHA256 |
|
36 | # 64 bytes for SHA256 | |
37 | _lfsre = re.compile(br'\A[a-f0-9]{64}\Z') |
|
37 | _lfsre = re.compile(br'\A[a-f0-9]{64}\Z') | |
38 |
|
38 | |||
39 | class lfsvfs(vfsmod.vfs): |
|
39 | class lfsvfs(vfsmod.vfs): | |
40 | def join(self, path): |
|
40 | def join(self, path): | |
41 | """split the path at first two characters, like: XX/XXXXX...""" |
|
41 | """split the path at first two characters, like: XX/XXXXX...""" | |
42 | if not _lfsre.match(path): |
|
42 | if not _lfsre.match(path): | |
43 | raise error.ProgrammingError('unexpected lfs path: %s' % path) |
|
43 | raise error.ProgrammingError('unexpected lfs path: %s' % path) | |
44 | return super(lfsvfs, self).join(path[0:2], path[2:]) |
|
44 | return super(lfsvfs, self).join(path[0:2], path[2:]) | |
45 |
|
45 | |||
46 | def walk(self, path=None, onerror=None): |
|
46 | def walk(self, path=None, onerror=None): | |
47 | """Yield (dirpath, [], oids) tuple for blobs under path |
|
47 | """Yield (dirpath, [], oids) tuple for blobs under path | |
48 |
|
48 | |||
49 | Oids only exist in the root of this vfs, so dirpath is always ''. |
|
49 | Oids only exist in the root of this vfs, so dirpath is always ''. | |
50 | """ |
|
50 | """ | |
51 | root = os.path.normpath(self.base) |
|
51 | root = os.path.normpath(self.base) | |
52 | # when dirpath == root, dirpath[prefixlen:] becomes empty |
|
52 | # when dirpath == root, dirpath[prefixlen:] becomes empty | |
53 | # because len(dirpath) < prefixlen. |
|
53 | # because len(dirpath) < prefixlen. | |
54 | prefixlen = len(pathutil.normasprefix(root)) |
|
54 | prefixlen = len(pathutil.normasprefix(root)) | |
55 | oids = [] |
|
55 | oids = [] | |
56 |
|
56 | |||
57 | for dirpath, dirs, files in os.walk(self.reljoin(self.base, path or ''), |
|
57 | for dirpath, dirs, files in os.walk(self.reljoin(self.base, path or ''), | |
58 | onerror=onerror): |
|
58 | onerror=onerror): | |
59 | dirpath = dirpath[prefixlen:] |
|
59 | dirpath = dirpath[prefixlen:] | |
60 |
|
60 | |||
61 | # Silently skip unexpected files and directories |
|
61 | # Silently skip unexpected files and directories | |
62 | if len(dirpath) == 2: |
|
62 | if len(dirpath) == 2: | |
63 | oids.extend([dirpath + f for f in files |
|
63 | oids.extend([dirpath + f for f in files | |
64 | if _lfsre.match(dirpath + f)]) |
|
64 | if _lfsre.match(dirpath + f)]) | |
65 |
|
65 | |||
66 | yield ('', [], oids) |
|
66 | yield ('', [], oids) | |
67 |
|
67 | |||
68 | class nullvfs(lfsvfs): |
|
68 | class nullvfs(lfsvfs): | |
69 | def __init__(self): |
|
69 | def __init__(self): | |
70 | pass |
|
70 | pass | |
71 |
|
71 | |||
72 | def exists(self, oid): |
|
72 | def exists(self, oid): | |
73 | return False |
|
73 | return False | |
74 |
|
74 | |||
75 | def read(self, oid): |
|
75 | def read(self, oid): | |
76 | # store.read() calls into here if the blob doesn't exist in its |
|
76 | # store.read() calls into here if the blob doesn't exist in its | |
77 | # self.vfs. Raise the same error as a normal vfs when asked to read a |
|
77 | # self.vfs. Raise the same error as a normal vfs when asked to read a | |
78 | # file that doesn't exist. The only difference is the full file path |
|
78 | # file that doesn't exist. The only difference is the full file path | |
79 | # isn't available in the error. |
|
79 | # isn't available in the error. | |
80 | raise IOError(errno.ENOENT, '%s: No such file or directory' % oid) |
|
80 | raise IOError(errno.ENOENT, '%s: No such file or directory' % oid) | |
81 |
|
81 | |||
82 | def walk(self, path=None, onerror=None): |
|
82 | def walk(self, path=None, onerror=None): | |
83 | return ('', [], []) |
|
83 | return ('', [], []) | |
84 |
|
84 | |||
85 | def write(self, oid, data): |
|
85 | def write(self, oid, data): | |
86 | pass |
|
86 | pass | |
87 |
|
87 | |||
88 | class filewithprogress(object): |
|
88 | class filewithprogress(object): | |
89 | """a file-like object that supports __len__ and read. |
|
89 | """a file-like object that supports __len__ and read. | |
90 |
|
90 | |||
91 | Useful to provide progress information for how many bytes are read. |
|
91 | Useful to provide progress information for how many bytes are read. | |
92 | """ |
|
92 | """ | |
93 |
|
93 | |||
94 | def __init__(self, fp, callback): |
|
94 | def __init__(self, fp, callback): | |
95 | self._fp = fp |
|
95 | self._fp = fp | |
96 | self._callback = callback # func(readsize) |
|
96 | self._callback = callback # func(readsize) | |
97 | fp.seek(0, os.SEEK_END) |
|
97 | fp.seek(0, os.SEEK_END) | |
98 | self._len = fp.tell() |
|
98 | self._len = fp.tell() | |
99 | fp.seek(0) |
|
99 | fp.seek(0) | |
100 |
|
100 | |||
101 | def __len__(self): |
|
101 | def __len__(self): | |
102 | return self._len |
|
102 | return self._len | |
103 |
|
103 | |||
104 | def read(self, size): |
|
104 | def read(self, size): | |
105 | if self._fp is None: |
|
105 | if self._fp is None: | |
106 | return b'' |
|
106 | return b'' | |
107 | data = self._fp.read(size) |
|
107 | data = self._fp.read(size) | |
108 | if data: |
|
108 | if data: | |
109 | if self._callback: |
|
109 | if self._callback: | |
110 | self._callback(len(data)) |
|
110 | self._callback(len(data)) | |
111 | else: |
|
111 | else: | |
112 | self._fp.close() |
|
112 | self._fp.close() | |
113 | self._fp = None |
|
113 | self._fp = None | |
114 | return data |
|
114 | return data | |
115 |
|
115 | |||
116 | class local(object): |
|
116 | class local(object): | |
117 | """Local blobstore for large file contents. |
|
117 | """Local blobstore for large file contents. | |
118 |
|
118 | |||
119 | This blobstore is used both as a cache and as a staging area for large blobs |
|
119 | This blobstore is used both as a cache and as a staging area for large blobs | |
120 | to be uploaded to the remote blobstore. |
|
120 | to be uploaded to the remote blobstore. | |
121 | """ |
|
121 | """ | |
122 |
|
122 | |||
123 | def __init__(self, repo): |
|
123 | def __init__(self, repo): | |
124 | fullpath = repo.svfs.join('lfs/objects') |
|
124 | fullpath = repo.svfs.join('lfs/objects') | |
125 | self.vfs = lfsvfs(fullpath) |
|
125 | self.vfs = lfsvfs(fullpath) | |
126 |
|
126 | |||
127 | if repo.ui.configbool('experimental', 'lfs.disableusercache'): |
|
127 | if repo.ui.configbool('experimental', 'lfs.disableusercache'): | |
128 | self.cachevfs = nullvfs() |
|
128 | self.cachevfs = nullvfs() | |
129 | else: |
|
129 | else: | |
130 | usercache = lfutil._usercachedir(repo.ui, 'lfs') |
|
130 | usercache = lfutil._usercachedir(repo.ui, 'lfs') | |
131 | self.cachevfs = lfsvfs(usercache) |
|
131 | self.cachevfs = lfsvfs(usercache) | |
132 | self.ui = repo.ui |
|
132 | self.ui = repo.ui | |
133 |
|
133 | |||
134 | def open(self, oid): |
|
134 | def open(self, oid): | |
135 | """Open a read-only file descriptor to the named blob, in either the |
|
135 | """Open a read-only file descriptor to the named blob, in either the | |
136 | usercache or the local store.""" |
|
136 | usercache or the local store.""" | |
137 | # The usercache is the most likely place to hold the file. Commit will |
|
137 | # The usercache is the most likely place to hold the file. Commit will | |
138 | # write to both it and the local store, as will anything that downloads |
|
138 | # write to both it and the local store, as will anything that downloads | |
139 | # the blobs. However, things like clone without an update won't |
|
139 | # the blobs. However, things like clone without an update won't | |
140 | # populate the local store. For an init + push of a local clone, |
|
140 | # populate the local store. For an init + push of a local clone, | |
141 | # the usercache is the only place it _could_ be. If not present, the |
|
141 | # the usercache is the only place it _could_ be. If not present, the | |
142 | # missing file msg here will indicate the local repo, not the usercache. |
|
142 | # missing file msg here will indicate the local repo, not the usercache. | |
143 | if self.cachevfs.exists(oid): |
|
143 | if self.cachevfs.exists(oid): | |
144 | return self.cachevfs(oid, 'rb') |
|
144 | return self.cachevfs(oid, 'rb') | |
145 |
|
145 | |||
146 | return self.vfs(oid, 'rb') |
|
146 | return self.vfs(oid, 'rb') | |
147 |
|
147 | |||
148 | def download(self, oid, src): |
|
148 | def download(self, oid, src): | |
149 | """Read the blob from the remote source in chunks, verify the content, |
|
149 | """Read the blob from the remote source in chunks, verify the content, | |
150 | and write to this local blobstore.""" |
|
150 | and write to this local blobstore.""" | |
151 | sha256 = hashlib.sha256() |
|
151 | sha256 = hashlib.sha256() | |
152 |
|
152 | |||
153 | with self.vfs(oid, 'wb', atomictemp=True) as fp: |
|
153 | with self.vfs(oid, 'wb', atomictemp=True) as fp: | |
154 | for chunk in util.filechunkiter(src, size=1048576): |
|
154 | for chunk in util.filechunkiter(src, size=1048576): | |
155 | fp.write(chunk) |
|
155 | fp.write(chunk) | |
156 | sha256.update(chunk) |
|
156 | sha256.update(chunk) | |
157 |
|
157 | |||
158 | realoid = sha256.hexdigest() |
|
158 | realoid = sha256.hexdigest() | |
159 | if realoid != oid: |
|
159 | if realoid != oid: | |
160 | raise LfsCorruptionError(_('corrupt remote lfs object: %s') |
|
160 | raise LfsCorruptionError(_('corrupt remote lfs object: %s') | |
161 | % oid) |
|
161 | % oid) | |
162 |
|
162 | |||
163 | self._linktousercache(oid) |
|
163 | self._linktousercache(oid) | |
164 |
|
164 | |||
165 | def write(self, oid, data): |
|
165 | def write(self, oid, data): | |
166 | """Write blob to local blobstore. |
|
166 | """Write blob to local blobstore. | |
167 |
|
167 | |||
168 | This should only be called from the filelog during a commit or similar. |
|
168 | This should only be called from the filelog during a commit or similar. | |
169 | As such, there is no need to verify the data. Imports from a remote |
|
169 | As such, there is no need to verify the data. Imports from a remote | |
170 | store must use ``download()`` instead.""" |
|
170 | store must use ``download()`` instead.""" | |
171 | with self.vfs(oid, 'wb', atomictemp=True) as fp: |
|
171 | with self.vfs(oid, 'wb', atomictemp=True) as fp: | |
172 | fp.write(data) |
|
172 | fp.write(data) | |
173 |
|
173 | |||
174 | self._linktousercache(oid) |
|
174 | self._linktousercache(oid) | |
175 |
|
175 | |||
176 | def linkfromusercache(self, oid): |
|
176 | def linkfromusercache(self, oid): | |
177 | """Link blobs found in the user cache into this store. |
|
177 | """Link blobs found in the user cache into this store. | |
178 |
|
178 | |||
179 | The server module needs to do this when it lets the client know not to |
|
179 | The server module needs to do this when it lets the client know not to | |
180 | upload the blob, to ensure it is always available in this store. |
|
180 | upload the blob, to ensure it is always available in this store. | |
181 | Normally this is done implicitly when the client reads or writes the |
|
181 | Normally this is done implicitly when the client reads or writes the | |
182 | blob, but that doesn't happen when the server tells the client that it |
|
182 | blob, but that doesn't happen when the server tells the client that it | |
183 | already has the blob. |
|
183 | already has the blob. | |
184 | """ |
|
184 | """ | |
185 | if (not isinstance(self.cachevfs, nullvfs) |
|
185 | if (not isinstance(self.cachevfs, nullvfs) | |
186 | and not self.vfs.exists(oid)): |
|
186 | and not self.vfs.exists(oid)): | |
187 | self.ui.note(_('lfs: found %s in the usercache\n') % oid) |
|
187 | self.ui.note(_('lfs: found %s in the usercache\n') % oid) | |
188 | lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) |
|
188 | lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) | |
189 |
|
189 | |||
190 | def _linktousercache(self, oid): |
|
190 | def _linktousercache(self, oid): | |
191 | # XXX: should we verify the content of the cache, and hardlink back to |
|
191 | # XXX: should we verify the content of the cache, and hardlink back to | |
192 | # the local store on success, but truncate, write and link on failure? |
|
192 | # the local store on success, but truncate, write and link on failure? | |
193 | if (not self.cachevfs.exists(oid) |
|
193 | if (not self.cachevfs.exists(oid) | |
194 | and not isinstance(self.cachevfs, nullvfs)): |
|
194 | and not isinstance(self.cachevfs, nullvfs)): | |
195 | self.ui.note(_('lfs: adding %s to the usercache\n') % oid) |
|
195 | self.ui.note(_('lfs: adding %s to the usercache\n') % oid) | |
196 | lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid)) |
|
196 | lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid)) | |
197 |
|
197 | |||
198 | def read(self, oid, verify=True): |
|
198 | def read(self, oid, verify=True): | |
199 | """Read blob from local blobstore.""" |
|
199 | """Read blob from local blobstore.""" | |
200 | if not self.vfs.exists(oid): |
|
200 | if not self.vfs.exists(oid): | |
201 | blob = self._read(self.cachevfs, oid, verify) |
|
201 | blob = self._read(self.cachevfs, oid, verify) | |
202 |
|
202 | |||
203 | # Even if revlog will verify the content, it needs to be verified |
|
203 | # Even if revlog will verify the content, it needs to be verified | |
204 | # now before making the hardlink to avoid propagating corrupt blobs. |
|
204 | # now before making the hardlink to avoid propagating corrupt blobs. | |
205 | # Don't abort if corruption is detected, because `hg verify` will |
|
205 | # Don't abort if corruption is detected, because `hg verify` will | |
206 | # give more useful info about the corruption- simply don't add the |
|
206 | # give more useful info about the corruption- simply don't add the | |
207 | # hardlink. |
|
207 | # hardlink. | |
208 | if verify or hashlib.sha256(blob).hexdigest() == oid: |
|
208 | if verify or hashlib.sha256(blob).hexdigest() == oid: | |
209 | self.ui.note(_('lfs: found %s in the usercache\n') % oid) |
|
209 | self.ui.note(_('lfs: found %s in the usercache\n') % oid) | |
210 | lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) |
|
210 | lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) | |
211 | else: |
|
211 | else: | |
212 | self.ui.note(_('lfs: found %s in the local lfs store\n') % oid) |
|
212 | self.ui.note(_('lfs: found %s in the local lfs store\n') % oid) | |
213 | blob = self._read(self.vfs, oid, verify) |
|
213 | blob = self._read(self.vfs, oid, verify) | |
214 | return blob |
|
214 | return blob | |
215 |
|
215 | |||
216 | def _read(self, vfs, oid, verify): |
|
216 | def _read(self, vfs, oid, verify): | |
217 | """Read blob (after verifying) from the given store""" |
|
217 | """Read blob (after verifying) from the given store""" | |
218 | blob = vfs.read(oid) |
|
218 | blob = vfs.read(oid) | |
219 | if verify: |
|
219 | if verify: | |
220 | _verify(oid, blob) |
|
220 | _verify(oid, blob) | |
221 | return blob |
|
221 | return blob | |
222 |
|
222 | |||
223 | def verify(self, oid): |
|
223 | def verify(self, oid): | |
224 | """Indicate whether or not the hash of the underlying file matches its |
|
224 | """Indicate whether or not the hash of the underlying file matches its | |
225 | name.""" |
|
225 | name.""" | |
226 | sha256 = hashlib.sha256() |
|
226 | sha256 = hashlib.sha256() | |
227 |
|
227 | |||
228 | with self.open(oid) as fp: |
|
228 | with self.open(oid) as fp: | |
229 | for chunk in util.filechunkiter(fp, size=1048576): |
|
229 | for chunk in util.filechunkiter(fp, size=1048576): | |
230 | sha256.update(chunk) |
|
230 | sha256.update(chunk) | |
231 |
|
231 | |||
232 | return oid == sha256.hexdigest() |
|
232 | return oid == sha256.hexdigest() | |
233 |
|
233 | |||
234 | def has(self, oid): |
|
234 | def has(self, oid): | |
235 | """Returns True if the local blobstore contains the requested blob, |
|
235 | """Returns True if the local blobstore contains the requested blob, | |
236 | False otherwise.""" |
|
236 | False otherwise.""" | |
237 | return self.cachevfs.exists(oid) or self.vfs.exists(oid) |
|
237 | return self.cachevfs.exists(oid) or self.vfs.exists(oid) | |
238 |
|
238 | |||
239 | def _urlerrorreason(urlerror): |
|
239 | def _urlerrorreason(urlerror): | |
240 | '''Create a friendly message for the given URLError to be used in an |
|
240 | '''Create a friendly message for the given URLError to be used in an | |
241 | LfsRemoteError message. |
|
241 | LfsRemoteError message. | |
242 | ''' |
|
242 | ''' | |
243 | inst = urlerror |
|
243 | inst = urlerror | |
244 |
|
244 | |||
245 | if isinstance(urlerror.reason, Exception): |
|
245 | if isinstance(urlerror.reason, Exception): | |
246 | inst = urlerror.reason |
|
246 | inst = urlerror.reason | |
247 |
|
247 | |||
248 | if util.safehasattr(inst, 'reason'): |
|
248 | if util.safehasattr(inst, 'reason'): | |
249 | try: # usually it is in the form (errno, strerror) |
|
249 | try: # usually it is in the form (errno, strerror) | |
250 | reason = inst.reason.args[1] |
|
250 | reason = inst.reason.args[1] | |
251 | except (AttributeError, IndexError): |
|
251 | except (AttributeError, IndexError): | |
252 | # it might be anything, for example a string |
|
252 | # it might be anything, for example a string | |
253 | reason = inst.reason |
|
253 | reason = inst.reason | |
254 | if isinstance(reason, pycompat.unicode): |
|
254 | if isinstance(reason, pycompat.unicode): | |
255 | # SSLError of Python 2.7.9 contains a unicode |
|
255 | # SSLError of Python 2.7.9 contains a unicode | |
256 | reason = encoding.unitolocal(reason) |
|
256 | reason = encoding.unitolocal(reason) | |
257 | return reason |
|
257 | return reason | |
258 | elif getattr(inst, "strerror", None): |
|
258 | elif getattr(inst, "strerror", None): | |
259 | return encoding.strtolocal(inst.strerror) |
|
259 | return encoding.strtolocal(inst.strerror) | |
260 | else: |
|
260 | else: | |
261 | return stringutil.forcebytestr(urlerror) |
|
261 | return stringutil.forcebytestr(urlerror) | |
262 |
|
262 | |||
263 | class _gitlfsremote(object): |
|
263 | class _gitlfsremote(object): | |
264 |
|
264 | |||
265 | def __init__(self, repo, url): |
|
265 | def __init__(self, repo, url): | |
266 | ui = repo.ui |
|
266 | ui = repo.ui | |
267 | self.ui = ui |
|
267 | self.ui = ui | |
268 | baseurl, authinfo = url.authinfo() |
|
268 | baseurl, authinfo = url.authinfo() | |
269 | self.baseurl = baseurl.rstrip('/') |
|
269 | self.baseurl = baseurl.rstrip('/') | |
270 | useragent = repo.ui.config('experimental', 'lfs.user-agent') |
|
270 | useragent = repo.ui.config('experimental', 'lfs.user-agent') | |
271 | if not useragent: |
|
271 | if not useragent: | |
272 | useragent = 'git-lfs/2.3.4 (Mercurial %s)' % util.version() |
|
272 | useragent = 'git-lfs/2.3.4 (Mercurial %s)' % util.version() | |
273 | self.urlopener = urlmod.opener(ui, authinfo, useragent) |
|
273 | self.urlopener = urlmod.opener(ui, authinfo, useragent) | |
274 | self.retry = ui.configint('lfs', 'retry') |
|
274 | self.retry = ui.configint('lfs', 'retry') | |
275 |
|
275 | |||
276 | def writebatch(self, pointers, fromstore): |
|
276 | def writebatch(self, pointers, fromstore): | |
277 | """Batch upload from local to remote blobstore.""" |
|
277 | """Batch upload from local to remote blobstore.""" | |
278 | self._batch(_deduplicate(pointers), fromstore, 'upload') |
|
278 | self._batch(_deduplicate(pointers), fromstore, 'upload') | |
279 |
|
279 | |||
280 | def readbatch(self, pointers, tostore): |
|
280 | def readbatch(self, pointers, tostore): | |
281 | """Batch download from remote to local blostore.""" |
|
281 | """Batch download from remote to local blostore.""" | |
282 | self._batch(_deduplicate(pointers), tostore, 'download') |
|
282 | self._batch(_deduplicate(pointers), tostore, 'download') | |
283 |
|
283 | |||
284 | def _batchrequest(self, pointers, action): |
|
284 | def _batchrequest(self, pointers, action): | |
285 | """Get metadata about objects pointed by pointers for given action |
|
285 | """Get metadata about objects pointed by pointers for given action | |
286 |
|
286 | |||
287 | Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} |
|
287 | Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} | |
288 | See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md |
|
288 | See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md | |
289 | """ |
|
289 | """ | |
290 | objects = [{'oid': p.oid(), 'size': p.size()} for p in pointers] |
|
290 | objects = [{'oid': p.oid(), 'size': p.size()} for p in pointers] | |
291 | requestdata = json.dumps({ |
|
291 | requestdata = json.dumps({ | |
292 | 'objects': objects, |
|
292 | 'objects': objects, | |
293 | 'operation': action, |
|
293 | 'operation': action, | |
294 | }) |
|
294 | }) | |
295 | url = '%s/objects/batch' % self.baseurl |
|
295 | url = '%s/objects/batch' % self.baseurl | |
296 | batchreq = util.urlreq.request(url, data=requestdata) |
|
296 | batchreq = util.urlreq.request(url, data=requestdata) | |
297 | batchreq.add_header('Accept', 'application/vnd.git-lfs+json') |
|
297 | batchreq.add_header('Accept', 'application/vnd.git-lfs+json') | |
298 | batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json') |
|
298 | batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json') | |
299 | try: |
|
299 | try: | |
300 | rsp = self.urlopener.open(batchreq) |
|
300 | rsp = self.urlopener.open(batchreq) | |
301 | rawjson = rsp.read() |
|
301 | rawjson = rsp.read() | |
302 | except util.urlerr.httperror as ex: |
|
302 | except util.urlerr.httperror as ex: | |
303 | hints = { |
|
303 | hints = { | |
304 | 400: _('check that lfs serving is enabled on %s and "%s" is ' |
|
304 | 400: _('check that lfs serving is enabled on %s and "%s" is ' | |
305 | 'supported') % (self.baseurl, action), |
|
305 | 'supported') % (self.baseurl, action), | |
306 | 404: _('the "lfs.url" config may be used to override %s') |
|
306 | 404: _('the "lfs.url" config may be used to override %s') | |
307 | % self.baseurl, |
|
307 | % self.baseurl, | |
308 | } |
|
308 | } | |
309 | hint = hints.get(ex.code, _('api=%s, action=%s') % (url, action)) |
|
309 | hint = hints.get(ex.code, _('api=%s, action=%s') % (url, action)) | |
310 | raise LfsRemoteError(_('LFS HTTP error: %s') % ex, hint=hint) |
|
310 | raise LfsRemoteError(_('LFS HTTP error: %s') % ex, hint=hint) | |
311 | except util.urlerr.urlerror as ex: |
|
311 | except util.urlerr.urlerror as ex: | |
312 | hint = (_('the "lfs.url" config may be used to override %s') |
|
312 | hint = (_('the "lfs.url" config may be used to override %s') | |
313 | % self.baseurl) |
|
313 | % self.baseurl) | |
314 | raise LfsRemoteError(_('LFS error: %s') % _urlerrorreason(ex), |
|
314 | raise LfsRemoteError(_('LFS error: %s') % _urlerrorreason(ex), | |
315 | hint=hint) |
|
315 | hint=hint) | |
316 | try: |
|
316 | try: | |
317 | response = json.loads(rawjson) |
|
317 | response = json.loads(rawjson) | |
318 | except ValueError: |
|
318 | except ValueError: | |
319 | raise LfsRemoteError(_('LFS server returns invalid JSON: %s') |
|
319 | raise LfsRemoteError(_('LFS server returns invalid JSON: %s') | |
320 | % rawjson) |
|
320 | % rawjson) | |
321 |
|
321 | |||
322 | if self.ui.debugflag: |
|
322 | if self.ui.debugflag: | |
323 | self.ui.debug('Status: %d\n' % rsp.status) |
|
323 | self.ui.debug('Status: %d\n' % rsp.status) | |
324 | # lfs-test-server and hg serve return headers in different order |
|
324 | # lfs-test-server and hg serve return headers in different order | |
325 | self.ui.debug('%s\n' |
|
325 | self.ui.debug('%s\n' | |
326 | % '\n'.join(sorted(str(rsp.info()).splitlines()))) |
|
326 | % '\n'.join(sorted(str(rsp.info()).splitlines()))) | |
327 |
|
327 | |||
328 | if 'objects' in response: |
|
328 | if 'objects' in response: | |
329 | response['objects'] = sorted(response['objects'], |
|
329 | response['objects'] = sorted(response['objects'], | |
330 | key=lambda p: p['oid']) |
|
330 | key=lambda p: p['oid']) | |
331 | self.ui.debug('%s\n' |
|
331 | self.ui.debug('%s\n' | |
332 | % json.dumps(response, indent=2, |
|
332 | % json.dumps(response, indent=2, | |
333 | separators=('', ': '), sort_keys=True)) |
|
333 | separators=('', ': '), sort_keys=True)) | |
334 |
|
334 | |||
335 | return response |
|
335 | return response | |
336 |
|
336 | |||
337 | def _checkforservererror(self, pointers, responses, action): |
|
337 | def _checkforservererror(self, pointers, responses, action): | |
338 | """Scans errors from objects |
|
338 | """Scans errors from objects | |
339 |
|
339 | |||
340 | Raises LfsRemoteError if any objects have an error""" |
|
340 | Raises LfsRemoteError if any objects have an error""" | |
341 | for response in responses: |
|
341 | for response in responses: | |
342 | # The server should return 404 when objects cannot be found. Some |
|
342 | # The server should return 404 when objects cannot be found. Some | |
343 | # server implementation (ex. lfs-test-server) does not set "error" |
|
343 | # server implementation (ex. lfs-test-server) does not set "error" | |
344 | # but just removes "download" from "actions". Treat that case |
|
344 | # but just removes "download" from "actions". Treat that case | |
345 | # as the same as 404 error. |
|
345 | # as the same as 404 error. | |
346 | if 'error' not in response: |
|
346 | if 'error' not in response: | |
347 | if (action == 'download' |
|
347 | if (action == 'download' | |
348 | and action not in response.get('actions', [])): |
|
348 | and action not in response.get('actions', [])): | |
349 | code = 404 |
|
349 | code = 404 | |
350 | else: |
|
350 | else: | |
351 | continue |
|
351 | continue | |
352 | else: |
|
352 | else: | |
353 | # An error dict without a code doesn't make much sense, so |
|
353 | # An error dict without a code doesn't make much sense, so | |
354 | # treat as a server error. |
|
354 | # treat as a server error. | |
355 | code = response.get('error').get('code', 500) |
|
355 | code = response.get('error').get('code', 500) | |
356 |
|
356 | |||
357 | ptrmap = {p.oid(): p for p in pointers} |
|
357 | ptrmap = {p.oid(): p for p in pointers} | |
358 | p = ptrmap.get(response['oid'], None) |
|
358 | p = ptrmap.get(response['oid'], None) | |
359 | if p: |
|
359 | if p: | |
360 | filename = getattr(p, 'filename', 'unknown') |
|
360 | filename = getattr(p, 'filename', 'unknown') | |
361 | errors = { |
|
361 | errors = { | |
362 | 404: 'The object does not exist', |
|
362 | 404: 'The object does not exist', | |
363 | 410: 'The object was removed by the owner', |
|
363 | 410: 'The object was removed by the owner', | |
364 | 422: 'Validation error', |
|
364 | 422: 'Validation error', | |
365 | 500: 'Internal server error', |
|
365 | 500: 'Internal server error', | |
366 | } |
|
366 | } | |
367 | msg = errors.get(code, 'status code %d' % code) |
|
367 | msg = errors.get(code, 'status code %d' % code) | |
368 | raise LfsRemoteError(_('LFS server error for "%s": %s') |
|
368 | raise LfsRemoteError(_('LFS server error for "%s": %s') | |
369 | % (filename, msg)) |
|
369 | % (filename, msg)) | |
370 | else: |
|
370 | else: | |
371 | raise LfsRemoteError( |
|
371 | raise LfsRemoteError( | |
372 | _('LFS server error. Unsolicited response for oid %s') |
|
372 | _('LFS server error. Unsolicited response for oid %s') | |
373 | % response['oid']) |
|
373 | % response['oid']) | |
374 |
|
374 | |||
375 | def _extractobjects(self, response, pointers, action): |
|
375 | def _extractobjects(self, response, pointers, action): | |
376 | """extract objects from response of the batch API |
|
376 | """extract objects from response of the batch API | |
377 |
|
377 | |||
378 | response: parsed JSON object returned by batch API |
|
378 | response: parsed JSON object returned by batch API | |
379 | return response['objects'] filtered by action |
|
379 | return response['objects'] filtered by action | |
380 | raise if any object has an error |
|
380 | raise if any object has an error | |
381 | """ |
|
381 | """ | |
382 | # Scan errors from objects - fail early |
|
382 | # Scan errors from objects - fail early | |
383 | objects = response.get('objects', []) |
|
383 | objects = response.get('objects', []) | |
384 | self._checkforservererror(pointers, objects, action) |
|
384 | self._checkforservererror(pointers, objects, action) | |
385 |
|
385 | |||
386 | # Filter objects with given action. Practically, this skips uploading |
|
386 | # Filter objects with given action. Practically, this skips uploading | |
387 | # objects which exist in the server. |
|
387 | # objects which exist in the server. | |
388 | filteredobjects = [o for o in objects if action in o.get('actions', [])] |
|
388 | filteredobjects = [o for o in objects if action in o.get('actions', [])] | |
389 |
|
389 | |||
390 | return filteredobjects |
|
390 | return filteredobjects | |
391 |
|
391 | |||
392 | def _basictransfer(self, obj, action, localstore): |
|
392 | def _basictransfer(self, obj, action, localstore): | |
393 | """Download or upload a single object using basic transfer protocol |
|
393 | """Download or upload a single object using basic transfer protocol | |
394 |
|
394 | |||
395 | obj: dict, an object description returned by batch API |
|
395 | obj: dict, an object description returned by batch API | |
396 | action: string, one of ['upload', 'download'] |
|
396 | action: string, one of ['upload', 'download'] | |
397 | localstore: blobstore.local |
|
397 | localstore: blobstore.local | |
398 |
|
398 | |||
399 | See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\ |
|
399 | See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\ | |
400 | basic-transfers.md |
|
400 | basic-transfers.md | |
401 | """ |
|
401 | """ | |
402 | oid = pycompat.bytestr(obj['oid']) |
|
402 | oid = pycompat.bytestr(obj['oid']) | |
403 |
|
403 | |||
404 | href = pycompat.bytestr(obj['actions'][action].get('href')) |
|
404 | href = pycompat.bytestr(obj['actions'][action].get('href')) | |
405 | headers = obj['actions'][action].get('header', {}).items() |
|
405 | headers = obj['actions'][action].get('header', {}).items() | |
406 |
|
406 | |||
407 | request = util.urlreq.request(href) |
|
407 | request = util.urlreq.request(href) | |
408 | if action == 'upload': |
|
408 | if action == 'upload': | |
409 | # If uploading blobs, read data from local blobstore. |
|
409 | # If uploading blobs, read data from local blobstore. | |
410 | if not localstore.verify(oid): |
|
410 | if not localstore.verify(oid): | |
411 | raise error.Abort(_('detected corrupt lfs object: %s') % oid, |
|
411 | raise error.Abort(_('detected corrupt lfs object: %s') % oid, | |
412 | hint=_('run hg verify')) |
|
412 | hint=_('run hg verify')) | |
413 | request.data = filewithprogress(localstore.open(oid), None) |
|
413 | request.data = filewithprogress(localstore.open(oid), None) | |
414 | request.get_method = lambda: 'PUT' |
|
414 | request.get_method = lambda: 'PUT' | |
415 | request.add_header('Content-Type', 'application/octet-stream') |
|
415 | request.add_header('Content-Type', 'application/octet-stream') | |
416 |
|
416 | |||
417 | for k, v in headers: |
|
417 | for k, v in headers: | |
418 | request.add_header(k, v) |
|
418 | request.add_header(k, v) | |
419 |
|
419 | |||
420 | response = b'' |
|
420 | response = b'' | |
421 | try: |
|
421 | try: | |
422 | req = self.urlopener.open(request) |
|
422 | req = self.urlopener.open(request) | |
423 |
|
423 | |||
424 | if self.ui.debugflag: |
|
424 | if self.ui.debugflag: | |
425 | self.ui.debug('Status: %d\n' % req.status) |
|
425 | self.ui.debug('Status: %d\n' % req.status) | |
426 | # lfs-test-server and hg serve return headers in different order |
|
426 | # lfs-test-server and hg serve return headers in different order | |
427 | self.ui.debug('%s\n' |
|
427 | self.ui.debug('%s\n' | |
428 | % '\n'.join(sorted(str(req.info()).splitlines()))) |
|
428 | % '\n'.join(sorted(str(req.info()).splitlines()))) | |
429 |
|
429 | |||
430 | if action == 'download': |
|
430 | if action == 'download': | |
431 | # If downloading blobs, store downloaded data to local blobstore |
|
431 | # If downloading blobs, store downloaded data to local blobstore | |
432 | localstore.download(oid, req) |
|
432 | localstore.download(oid, req) | |
433 | else: |
|
433 | else: | |
434 | while True: |
|
434 | while True: | |
435 | data = req.read(1048576) |
|
435 | data = req.read(1048576) | |
436 | if not data: |
|
436 | if not data: | |
437 | break |
|
437 | break | |
438 | response += data |
|
438 | response += data | |
439 | if response: |
|
439 | if response: | |
440 | self.ui.debug('lfs %s response: %s' % (action, response)) |
|
440 | self.ui.debug('lfs %s response: %s' % (action, response)) | |
441 | except util.urlerr.httperror as ex: |
|
441 | except util.urlerr.httperror as ex: | |
442 | if self.ui.debugflag: |
|
442 | if self.ui.debugflag: | |
443 | self.ui.debug('%s: %s\n' % (oid, ex.read())) |
|
443 | self.ui.debug('%s: %s\n' % (oid, ex.read())) | |
444 | raise LfsRemoteError(_('HTTP error: %s (oid=%s, action=%s)') |
|
444 | raise LfsRemoteError(_('LFS HTTP error: %s (oid=%s, action=%s)') | |
445 | % (ex, oid, action)) |
|
445 | % (ex, oid, action)) | |
446 | except util.urlerr.urlerror as ex: |
|
446 | except util.urlerr.urlerror as ex: | |
447 | hint = (_('attempted connection to %s') |
|
447 | hint = (_('attempted connection to %s') | |
448 | % util.urllibcompat.getfullurl(request)) |
|
448 | % util.urllibcompat.getfullurl(request)) | |
449 | raise LfsRemoteError(_('LFS error: %s') % _urlerrorreason(ex), |
|
449 | raise LfsRemoteError(_('LFS error: %s') % _urlerrorreason(ex), | |
450 | hint=hint) |
|
450 | hint=hint) | |
451 |
|
451 | |||
452 | def _batch(self, pointers, localstore, action): |
|
452 | def _batch(self, pointers, localstore, action): | |
453 | if action not in ['upload', 'download']: |
|
453 | if action not in ['upload', 'download']: | |
454 | raise error.ProgrammingError('invalid Git-LFS action: %s' % action) |
|
454 | raise error.ProgrammingError('invalid Git-LFS action: %s' % action) | |
455 |
|
455 | |||
456 | response = self._batchrequest(pointers, action) |
|
456 | response = self._batchrequest(pointers, action) | |
457 | objects = self._extractobjects(response, pointers, action) |
|
457 | objects = self._extractobjects(response, pointers, action) | |
458 | total = sum(x.get('size', 0) for x in objects) |
|
458 | total = sum(x.get('size', 0) for x in objects) | |
459 | sizes = {} |
|
459 | sizes = {} | |
460 | for obj in objects: |
|
460 | for obj in objects: | |
461 | sizes[obj.get('oid')] = obj.get('size', 0) |
|
461 | sizes[obj.get('oid')] = obj.get('size', 0) | |
462 | topic = {'upload': _('lfs uploading'), |
|
462 | topic = {'upload': _('lfs uploading'), | |
463 | 'download': _('lfs downloading')}[action] |
|
463 | 'download': _('lfs downloading')}[action] | |
464 | if len(objects) > 1: |
|
464 | if len(objects) > 1: | |
465 | self.ui.note(_('lfs: need to transfer %d objects (%s)\n') |
|
465 | self.ui.note(_('lfs: need to transfer %d objects (%s)\n') | |
466 | % (len(objects), util.bytecount(total))) |
|
466 | % (len(objects), util.bytecount(total))) | |
467 |
|
467 | |||
468 | def transfer(chunk): |
|
468 | def transfer(chunk): | |
469 | for obj in chunk: |
|
469 | for obj in chunk: | |
470 | objsize = obj.get('size', 0) |
|
470 | objsize = obj.get('size', 0) | |
471 | if self.ui.verbose: |
|
471 | if self.ui.verbose: | |
472 | if action == 'download': |
|
472 | if action == 'download': | |
473 | msg = _('lfs: downloading %s (%s)\n') |
|
473 | msg = _('lfs: downloading %s (%s)\n') | |
474 | elif action == 'upload': |
|
474 | elif action == 'upload': | |
475 | msg = _('lfs: uploading %s (%s)\n') |
|
475 | msg = _('lfs: uploading %s (%s)\n') | |
476 | self.ui.note(msg % (obj.get('oid'), |
|
476 | self.ui.note(msg % (obj.get('oid'), | |
477 | util.bytecount(objsize))) |
|
477 | util.bytecount(objsize))) | |
478 | retry = self.retry |
|
478 | retry = self.retry | |
479 | while True: |
|
479 | while True: | |
480 | try: |
|
480 | try: | |
481 | self._basictransfer(obj, action, localstore) |
|
481 | self._basictransfer(obj, action, localstore) | |
482 | yield 1, obj.get('oid') |
|
482 | yield 1, obj.get('oid') | |
483 | break |
|
483 | break | |
484 | except socket.error as ex: |
|
484 | except socket.error as ex: | |
485 | if retry > 0: |
|
485 | if retry > 0: | |
486 | self.ui.note( |
|
486 | self.ui.note( | |
487 | _('lfs: failed: %r (remaining retry %d)\n') |
|
487 | _('lfs: failed: %r (remaining retry %d)\n') | |
488 | % (ex, retry)) |
|
488 | % (ex, retry)) | |
489 | retry -= 1 |
|
489 | retry -= 1 | |
490 | continue |
|
490 | continue | |
491 | raise |
|
491 | raise | |
492 |
|
492 | |||
493 | # Until https multiplexing gets sorted out |
|
493 | # Until https multiplexing gets sorted out | |
494 | if self.ui.configbool('experimental', 'lfs.worker-enable'): |
|
494 | if self.ui.configbool('experimental', 'lfs.worker-enable'): | |
495 | oids = worker.worker(self.ui, 0.1, transfer, (), |
|
495 | oids = worker.worker(self.ui, 0.1, transfer, (), | |
496 | sorted(objects, key=lambda o: o.get('oid'))) |
|
496 | sorted(objects, key=lambda o: o.get('oid'))) | |
497 | else: |
|
497 | else: | |
498 | oids = transfer(sorted(objects, key=lambda o: o.get('oid'))) |
|
498 | oids = transfer(sorted(objects, key=lambda o: o.get('oid'))) | |
499 |
|
499 | |||
500 | with self.ui.makeprogress(topic, total=total) as progress: |
|
500 | with self.ui.makeprogress(topic, total=total) as progress: | |
501 | progress.update(0) |
|
501 | progress.update(0) | |
502 | processed = 0 |
|
502 | processed = 0 | |
503 | blobs = 0 |
|
503 | blobs = 0 | |
504 | for _one, oid in oids: |
|
504 | for _one, oid in oids: | |
505 | processed += sizes[oid] |
|
505 | processed += sizes[oid] | |
506 | blobs += 1 |
|
506 | blobs += 1 | |
507 | progress.update(processed) |
|
507 | progress.update(processed) | |
508 | self.ui.note(_('lfs: processed: %s\n') % oid) |
|
508 | self.ui.note(_('lfs: processed: %s\n') % oid) | |
509 |
|
509 | |||
510 | if blobs > 0: |
|
510 | if blobs > 0: | |
511 | if action == 'upload': |
|
511 | if action == 'upload': | |
512 | self.ui.status(_('lfs: uploaded %d files (%s)\n') |
|
512 | self.ui.status(_('lfs: uploaded %d files (%s)\n') | |
513 | % (blobs, util.bytecount(processed))) |
|
513 | % (blobs, util.bytecount(processed))) | |
514 | elif action == 'download': |
|
514 | elif action == 'download': | |
515 | self.ui.status(_('lfs: downloaded %d files (%s)\n') |
|
515 | self.ui.status(_('lfs: downloaded %d files (%s)\n') | |
516 | % (blobs, util.bytecount(processed))) |
|
516 | % (blobs, util.bytecount(processed))) | |
517 |
|
517 | |||
518 | def __del__(self): |
|
518 | def __del__(self): | |
519 | # copied from mercurial/httppeer.py |
|
519 | # copied from mercurial/httppeer.py | |
520 | urlopener = getattr(self, 'urlopener', None) |
|
520 | urlopener = getattr(self, 'urlopener', None) | |
521 | if urlopener: |
|
521 | if urlopener: | |
522 | for h in urlopener.handlers: |
|
522 | for h in urlopener.handlers: | |
523 | h.close() |
|
523 | h.close() | |
524 | getattr(h, "close_all", lambda : None)() |
|
524 | getattr(h, "close_all", lambda : None)() | |
525 |
|
525 | |||
526 | class _dummyremote(object): |
|
526 | class _dummyremote(object): | |
527 | """Dummy store storing blobs to temp directory.""" |
|
527 | """Dummy store storing blobs to temp directory.""" | |
528 |
|
528 | |||
529 | def __init__(self, repo, url): |
|
529 | def __init__(self, repo, url): | |
530 | fullpath = repo.vfs.join('lfs', url.path) |
|
530 | fullpath = repo.vfs.join('lfs', url.path) | |
531 | self.vfs = lfsvfs(fullpath) |
|
531 | self.vfs = lfsvfs(fullpath) | |
532 |
|
532 | |||
533 | def writebatch(self, pointers, fromstore): |
|
533 | def writebatch(self, pointers, fromstore): | |
534 | for p in _deduplicate(pointers): |
|
534 | for p in _deduplicate(pointers): | |
535 | content = fromstore.read(p.oid(), verify=True) |
|
535 | content = fromstore.read(p.oid(), verify=True) | |
536 | with self.vfs(p.oid(), 'wb', atomictemp=True) as fp: |
|
536 | with self.vfs(p.oid(), 'wb', atomictemp=True) as fp: | |
537 | fp.write(content) |
|
537 | fp.write(content) | |
538 |
|
538 | |||
539 | def readbatch(self, pointers, tostore): |
|
539 | def readbatch(self, pointers, tostore): | |
540 | for p in _deduplicate(pointers): |
|
540 | for p in _deduplicate(pointers): | |
541 | with self.vfs(p.oid(), 'rb') as fp: |
|
541 | with self.vfs(p.oid(), 'rb') as fp: | |
542 | tostore.download(p.oid(), fp) |
|
542 | tostore.download(p.oid(), fp) | |
543 |
|
543 | |||
544 | class _nullremote(object): |
|
544 | class _nullremote(object): | |
545 | """Null store storing blobs to /dev/null.""" |
|
545 | """Null store storing blobs to /dev/null.""" | |
546 |
|
546 | |||
547 | def __init__(self, repo, url): |
|
547 | def __init__(self, repo, url): | |
548 | pass |
|
548 | pass | |
549 |
|
549 | |||
550 | def writebatch(self, pointers, fromstore): |
|
550 | def writebatch(self, pointers, fromstore): | |
551 | pass |
|
551 | pass | |
552 |
|
552 | |||
553 | def readbatch(self, pointers, tostore): |
|
553 | def readbatch(self, pointers, tostore): | |
554 | pass |
|
554 | pass | |
555 |
|
555 | |||
556 | class _promptremote(object): |
|
556 | class _promptremote(object): | |
557 | """Prompt user to set lfs.url when accessed.""" |
|
557 | """Prompt user to set lfs.url when accessed.""" | |
558 |
|
558 | |||
559 | def __init__(self, repo, url): |
|
559 | def __init__(self, repo, url): | |
560 | pass |
|
560 | pass | |
561 |
|
561 | |||
562 | def writebatch(self, pointers, fromstore, ui=None): |
|
562 | def writebatch(self, pointers, fromstore, ui=None): | |
563 | self._prompt() |
|
563 | self._prompt() | |
564 |
|
564 | |||
565 | def readbatch(self, pointers, tostore, ui=None): |
|
565 | def readbatch(self, pointers, tostore, ui=None): | |
566 | self._prompt() |
|
566 | self._prompt() | |
567 |
|
567 | |||
568 | def _prompt(self): |
|
568 | def _prompt(self): | |
569 | raise error.Abort(_('lfs.url needs to be configured')) |
|
569 | raise error.Abort(_('lfs.url needs to be configured')) | |
570 |
|
570 | |||
571 | _storemap = { |
|
571 | _storemap = { | |
572 | 'https': _gitlfsremote, |
|
572 | 'https': _gitlfsremote, | |
573 | 'http': _gitlfsremote, |
|
573 | 'http': _gitlfsremote, | |
574 | 'file': _dummyremote, |
|
574 | 'file': _dummyremote, | |
575 | 'null': _nullremote, |
|
575 | 'null': _nullremote, | |
576 | None: _promptremote, |
|
576 | None: _promptremote, | |
577 | } |
|
577 | } | |
578 |
|
578 | |||
579 | def _deduplicate(pointers): |
|
579 | def _deduplicate(pointers): | |
580 | """Remove any duplicate oids that exist in the list""" |
|
580 | """Remove any duplicate oids that exist in the list""" | |
581 | reduced = util.sortdict() |
|
581 | reduced = util.sortdict() | |
582 | for p in pointers: |
|
582 | for p in pointers: | |
583 | reduced[p.oid()] = p |
|
583 | reduced[p.oid()] = p | |
584 | return reduced.values() |
|
584 | return reduced.values() | |
585 |
|
585 | |||
586 | def _verify(oid, content): |
|
586 | def _verify(oid, content): | |
587 | realoid = hashlib.sha256(content).hexdigest() |
|
587 | realoid = hashlib.sha256(content).hexdigest() | |
588 | if realoid != oid: |
|
588 | if realoid != oid: | |
589 | raise LfsCorruptionError(_('detected corrupt lfs object: %s') % oid, |
|
589 | raise LfsCorruptionError(_('detected corrupt lfs object: %s') % oid, | |
590 | hint=_('run hg verify')) |
|
590 | hint=_('run hg verify')) | |
591 |
|
591 | |||
592 | def remote(repo, remote=None): |
|
592 | def remote(repo, remote=None): | |
593 | """remotestore factory. return a store in _storemap depending on config |
|
593 | """remotestore factory. return a store in _storemap depending on config | |
594 |
|
594 | |||
595 | If ``lfs.url`` is specified, use that remote endpoint. Otherwise, try to |
|
595 | If ``lfs.url`` is specified, use that remote endpoint. Otherwise, try to | |
596 | infer the endpoint, based on the remote repository using the same path |
|
596 | infer the endpoint, based on the remote repository using the same path | |
597 | adjustments as git. As an extension, 'http' is supported as well so that |
|
597 | adjustments as git. As an extension, 'http' is supported as well so that | |
598 | ``hg serve`` works out of the box. |
|
598 | ``hg serve`` works out of the box. | |
599 |
|
599 | |||
600 | https://github.com/git-lfs/git-lfs/blob/master/docs/api/server-discovery.md |
|
600 | https://github.com/git-lfs/git-lfs/blob/master/docs/api/server-discovery.md | |
601 | """ |
|
601 | """ | |
602 | lfsurl = repo.ui.config('lfs', 'url') |
|
602 | lfsurl = repo.ui.config('lfs', 'url') | |
603 | url = util.url(lfsurl or '') |
|
603 | url = util.url(lfsurl or '') | |
604 | if lfsurl is None: |
|
604 | if lfsurl is None: | |
605 | if remote: |
|
605 | if remote: | |
606 | path = remote |
|
606 | path = remote | |
607 | elif util.safehasattr(repo, '_subtoppath'): |
|
607 | elif util.safehasattr(repo, '_subtoppath'): | |
608 | # The pull command sets this during the optional update phase, which |
|
608 | # The pull command sets this during the optional update phase, which | |
609 | # tells exactly where the pull originated, whether 'paths.default' |
|
609 | # tells exactly where the pull originated, whether 'paths.default' | |
610 | # or explicit. |
|
610 | # or explicit. | |
611 | path = repo._subtoppath |
|
611 | path = repo._subtoppath | |
612 | else: |
|
612 | else: | |
613 | # TODO: investigate 'paths.remote:lfsurl' style path customization, |
|
613 | # TODO: investigate 'paths.remote:lfsurl' style path customization, | |
614 | # and fall back to inferring from 'paths.remote' if unspecified. |
|
614 | # and fall back to inferring from 'paths.remote' if unspecified. | |
615 | path = repo.ui.config('paths', 'default') or '' |
|
615 | path = repo.ui.config('paths', 'default') or '' | |
616 |
|
616 | |||
617 | defaulturl = util.url(path) |
|
617 | defaulturl = util.url(path) | |
618 |
|
618 | |||
619 | # TODO: support local paths as well. |
|
619 | # TODO: support local paths as well. | |
620 | # TODO: consider the ssh -> https transformation that git applies |
|
620 | # TODO: consider the ssh -> https transformation that git applies | |
621 | if defaulturl.scheme in (b'http', b'https'): |
|
621 | if defaulturl.scheme in (b'http', b'https'): | |
622 | if defaulturl.path and defaulturl.path[:-1] != b'/': |
|
622 | if defaulturl.path and defaulturl.path[:-1] != b'/': | |
623 | defaulturl.path += b'/' |
|
623 | defaulturl.path += b'/' | |
624 | defaulturl.path = (defaulturl.path or b'') + b'.git/info/lfs' |
|
624 | defaulturl.path = (defaulturl.path or b'') + b'.git/info/lfs' | |
625 |
|
625 | |||
626 | url = util.url(bytes(defaulturl)) |
|
626 | url = util.url(bytes(defaulturl)) | |
627 | repo.ui.note(_('lfs: assuming remote store: %s\n') % url) |
|
627 | repo.ui.note(_('lfs: assuming remote store: %s\n') % url) | |
628 |
|
628 | |||
629 | scheme = url.scheme |
|
629 | scheme = url.scheme | |
630 | if scheme not in _storemap: |
|
630 | if scheme not in _storemap: | |
631 | raise error.Abort(_('lfs: unknown url scheme: %s') % scheme) |
|
631 | raise error.Abort(_('lfs: unknown url scheme: %s') % scheme) | |
632 | return _storemap[scheme](repo, url) |
|
632 | return _storemap[scheme](repo, url) | |
633 |
|
633 | |||
634 | class LfsRemoteError(error.StorageError): |
|
634 | class LfsRemoteError(error.StorageError): | |
635 | pass |
|
635 | pass | |
636 |
|
636 | |||
637 | class LfsCorruptionError(error.Abort): |
|
637 | class LfsCorruptionError(error.Abort): | |
638 | """Raised when a corrupt blob is detected, aborting an operation |
|
638 | """Raised when a corrupt blob is detected, aborting an operation | |
639 |
|
639 | |||
640 | It exists to allow specialized handling on the server side.""" |
|
640 | It exists to allow specialized handling on the server side.""" |
@@ -1,480 +1,480 b'' | |||||
1 | #require serve no-reposimplestore no-chg |
|
1 | #require serve no-reposimplestore no-chg | |
2 |
|
2 | |||
3 | $ cat >> $HGRCPATH <<EOF |
|
3 | $ cat >> $HGRCPATH <<EOF | |
4 | > [extensions] |
|
4 | > [extensions] | |
5 | > lfs= |
|
5 | > lfs= | |
6 | > [lfs] |
|
6 | > [lfs] | |
7 | > track=all() |
|
7 | > track=all() | |
8 | > [web] |
|
8 | > [web] | |
9 | > push_ssl = False |
|
9 | > push_ssl = False | |
10 | > allow-push = * |
|
10 | > allow-push = * | |
11 | > EOF |
|
11 | > EOF | |
12 |
|
12 | |||
13 | Serving LFS files can experimentally be turned off. The long term solution is |
|
13 | Serving LFS files can experimentally be turned off. The long term solution is | |
14 | to support the 'verify' action in both client and server, so that the server can |
|
14 | to support the 'verify' action in both client and server, so that the server can | |
15 | tell the client to store files elsewhere. |
|
15 | tell the client to store files elsewhere. | |
16 |
|
16 | |||
17 | $ hg init server |
|
17 | $ hg init server | |
18 | $ hg --config "lfs.usercache=$TESTTMP/servercache" \ |
|
18 | $ hg --config "lfs.usercache=$TESTTMP/servercache" \ | |
19 | > --config experimental.lfs.serve=False -R server serve -d \ |
|
19 | > --config experimental.lfs.serve=False -R server serve -d \ | |
20 | > -p $HGPORT --pid-file=hg.pid -A $TESTTMP/access.log -E $TESTTMP/errors.log |
|
20 | > -p $HGPORT --pid-file=hg.pid -A $TESTTMP/access.log -E $TESTTMP/errors.log | |
21 | $ cat hg.pid >> $DAEMON_PIDS |
|
21 | $ cat hg.pid >> $DAEMON_PIDS | |
22 |
|
22 | |||
23 | Uploads fail... |
|
23 | Uploads fail... | |
24 |
|
24 | |||
25 | $ hg init client |
|
25 | $ hg init client | |
26 | $ echo 'this-is-an-lfs-file' > client/lfs.bin |
|
26 | $ echo 'this-is-an-lfs-file' > client/lfs.bin | |
27 | $ hg -R client ci -Am 'initial commit' |
|
27 | $ hg -R client ci -Am 'initial commit' | |
28 | adding lfs.bin |
|
28 | adding lfs.bin | |
29 | $ hg -R client push http://localhost:$HGPORT |
|
29 | $ hg -R client push http://localhost:$HGPORT | |
30 | pushing to http://localhost:$HGPORT/ |
|
30 | pushing to http://localhost:$HGPORT/ | |
31 | searching for changes |
|
31 | searching for changes | |
32 | abort: LFS HTTP error: HTTP Error 400: no such method: .git! |
|
32 | abort: LFS HTTP error: HTTP Error 400: no such method: .git! | |
33 | (check that lfs serving is enabled on http://localhost:$HGPORT/.git/info/lfs and "upload" is supported) |
|
33 | (check that lfs serving is enabled on http://localhost:$HGPORT/.git/info/lfs and "upload" is supported) | |
34 | [255] |
|
34 | [255] | |
35 |
|
35 | |||
36 | ... so do a local push to make the data available. Remove the blob from the |
|
36 | ... so do a local push to make the data available. Remove the blob from the | |
37 | default cache, so it attempts to download. |
|
37 | default cache, so it attempts to download. | |
38 | $ hg --config "lfs.usercache=$TESTTMP/servercache" \ |
|
38 | $ hg --config "lfs.usercache=$TESTTMP/servercache" \ | |
39 | > --config "lfs.url=null://" \ |
|
39 | > --config "lfs.url=null://" \ | |
40 | > -R client push -q server |
|
40 | > -R client push -q server | |
41 | $ mv `hg config lfs.usercache` $TESTTMP/servercache |
|
41 | $ mv `hg config lfs.usercache` $TESTTMP/servercache | |
42 |
|
42 | |||
43 | Downloads fail... |
|
43 | Downloads fail... | |
44 |
|
44 | |||
45 | $ hg clone http://localhost:$HGPORT httpclone |
|
45 | $ hg clone http://localhost:$HGPORT httpclone | |
46 | (remote is using large file support (lfs); lfs will be enabled for this repository) |
|
46 | (remote is using large file support (lfs); lfs will be enabled for this repository) | |
47 | requesting all changes |
|
47 | requesting all changes | |
48 | adding changesets |
|
48 | adding changesets | |
49 | adding manifests |
|
49 | adding manifests | |
50 | adding file changes |
|
50 | adding file changes | |
51 | added 1 changesets with 1 changes to 1 files |
|
51 | added 1 changesets with 1 changes to 1 files | |
52 | new changesets 525251863cad |
|
52 | new changesets 525251863cad | |
53 | updating to branch default |
|
53 | updating to branch default | |
54 | abort: LFS HTTP error: HTTP Error 400: no such method: .git! |
|
54 | abort: LFS HTTP error: HTTP Error 400: no such method: .git! | |
55 | (check that lfs serving is enabled on http://localhost:$HGPORT/.git/info/lfs and "download" is supported) |
|
55 | (check that lfs serving is enabled on http://localhost:$HGPORT/.git/info/lfs and "download" is supported) | |
56 | [255] |
|
56 | [255] | |
57 |
|
57 | |||
58 | $ "$PYTHON" $RUNTESTDIR/killdaemons.py $DAEMON_PIDS |
|
58 | $ "$PYTHON" $RUNTESTDIR/killdaemons.py $DAEMON_PIDS | |
59 |
|
59 | |||
60 | $ cat $TESTTMP/access.log $TESTTMP/errors.log |
|
60 | $ cat $TESTTMP/access.log $TESTTMP/errors.log | |
61 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) |
|
61 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) | |
62 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D525251863cad618e55d483555f3d00a2ca99597e x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
62 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D525251863cad618e55d483555f3d00a2ca99597e x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
63 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
63 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
64 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
64 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
65 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 400 - (glob) |
|
65 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 400 - (glob) | |
66 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) |
|
66 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) | |
67 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
67 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
68 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bookmarks=1&bundlecaps=HG20%2Cbundle2%3DHG20%250Abookmarks%250Achangegroup%253D01%252C02%252C03%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Aphases%253Dheads%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps%250Arev-branch-cache%250Astream%253Dv2&cg=1&common=0000000000000000000000000000000000000000&heads=525251863cad618e55d483555f3d00a2ca99597e&listkeys=bookmarks&phases=1 x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
68 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bookmarks=1&bundlecaps=HG20%2Cbundle2%3DHG20%250Abookmarks%250Achangegroup%253D01%252C02%252C03%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Aphases%253Dheads%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps%250Arev-branch-cache%250Astream%253Dv2&cg=1&common=0000000000000000000000000000000000000000&heads=525251863cad618e55d483555f3d00a2ca99597e&listkeys=bookmarks&phases=1 x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
69 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 400 - (glob) |
|
69 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 400 - (glob) | |
70 |
|
70 | |||
71 | $ rm -f $TESTTMP/access.log $TESTTMP/errors.log |
|
71 | $ rm -f $TESTTMP/access.log $TESTTMP/errors.log | |
72 | $ hg --config "lfs.usercache=$TESTTMP/servercache" -R server serve -d \ |
|
72 | $ hg --config "lfs.usercache=$TESTTMP/servercache" -R server serve -d \ | |
73 | > -p $HGPORT --pid-file=hg.pid --prefix=subdir/mount/point \ |
|
73 | > -p $HGPORT --pid-file=hg.pid --prefix=subdir/mount/point \ | |
74 | > -A $TESTTMP/access.log -E $TESTTMP/errors.log |
|
74 | > -A $TESTTMP/access.log -E $TESTTMP/errors.log | |
75 | $ cat hg.pid >> $DAEMON_PIDS |
|
75 | $ cat hg.pid >> $DAEMON_PIDS | |
76 |
|
76 | |||
77 | Reasonable hint for a misconfigured blob server |
|
77 | Reasonable hint for a misconfigured blob server | |
78 |
|
78 | |||
79 | $ hg -R httpclone update default --config lfs.url=http://localhost:$HGPORT/missing |
|
79 | $ hg -R httpclone update default --config lfs.url=http://localhost:$HGPORT/missing | |
80 | abort: LFS HTTP error: HTTP Error 404: Not Found! |
|
80 | abort: LFS HTTP error: HTTP Error 404: Not Found! | |
81 | (the "lfs.url" config may be used to override http://localhost:$HGPORT/missing) |
|
81 | (the "lfs.url" config may be used to override http://localhost:$HGPORT/missing) | |
82 | [255] |
|
82 | [255] | |
83 |
|
83 | |||
84 | $ hg -R httpclone update default --config lfs.url=http://localhost:$HGPORT2/missing |
|
84 | $ hg -R httpclone update default --config lfs.url=http://localhost:$HGPORT2/missing | |
85 | abort: LFS error: *onnection *refused*! (glob) |
|
85 | abort: LFS error: *onnection *refused*! (glob) | |
86 | (the "lfs.url" config may be used to override http://localhost:$HGPORT2/missing) |
|
86 | (the "lfs.url" config may be used to override http://localhost:$HGPORT2/missing) | |
87 | [255] |
|
87 | [255] | |
88 |
|
88 | |||
89 | Blob URIs are correct when --prefix is used |
|
89 | Blob URIs are correct when --prefix is used | |
90 |
|
90 | |||
91 | $ hg clone --debug http://localhost:$HGPORT/subdir/mount/point cloned2 |
|
91 | $ hg clone --debug http://localhost:$HGPORT/subdir/mount/point cloned2 | |
92 | using http://localhost:$HGPORT/subdir/mount/point |
|
92 | using http://localhost:$HGPORT/subdir/mount/point | |
93 | sending capabilities command |
|
93 | sending capabilities command | |
94 | (remote is using large file support (lfs); lfs will be enabled for this repository) |
|
94 | (remote is using large file support (lfs); lfs will be enabled for this repository) | |
95 | query 1; heads |
|
95 | query 1; heads | |
96 | sending batch command |
|
96 | sending batch command | |
97 | requesting all changes |
|
97 | requesting all changes | |
98 | sending getbundle command |
|
98 | sending getbundle command | |
99 | bundle2-input-bundle: with-transaction |
|
99 | bundle2-input-bundle: with-transaction | |
100 | bundle2-input-part: "changegroup" (params: 1 mandatory 1 advisory) supported |
|
100 | bundle2-input-part: "changegroup" (params: 1 mandatory 1 advisory) supported | |
101 | adding changesets |
|
101 | adding changesets | |
102 | add changeset 525251863cad |
|
102 | add changeset 525251863cad | |
103 | adding manifests |
|
103 | adding manifests | |
104 | adding file changes |
|
104 | adding file changes | |
105 | adding lfs.bin revisions |
|
105 | adding lfs.bin revisions | |
106 | added 1 changesets with 1 changes to 1 files |
|
106 | added 1 changesets with 1 changes to 1 files | |
107 | bundle2-input-part: total payload size 648 |
|
107 | bundle2-input-part: total payload size 648 | |
108 | bundle2-input-part: "listkeys" (params: 1 mandatory) supported |
|
108 | bundle2-input-part: "listkeys" (params: 1 mandatory) supported | |
109 | bundle2-input-part: "phase-heads" supported |
|
109 | bundle2-input-part: "phase-heads" supported | |
110 | bundle2-input-part: total payload size 24 |
|
110 | bundle2-input-part: total payload size 24 | |
111 | bundle2-input-part: "cache:rev-branch-cache" (advisory) supported |
|
111 | bundle2-input-part: "cache:rev-branch-cache" (advisory) supported | |
112 | bundle2-input-part: total payload size 39 |
|
112 | bundle2-input-part: total payload size 39 | |
113 | bundle2-input-bundle: 3 parts total |
|
113 | bundle2-input-bundle: 3 parts total | |
114 | checking for updated bookmarks |
|
114 | checking for updated bookmarks | |
115 | updating the branch cache |
|
115 | updating the branch cache | |
116 | new changesets 525251863cad |
|
116 | new changesets 525251863cad | |
117 | updating to branch default |
|
117 | updating to branch default | |
118 | resolving manifests |
|
118 | resolving manifests | |
119 | branchmerge: False, force: False, partial: False |
|
119 | branchmerge: False, force: False, partial: False | |
120 | ancestor: 000000000000, local: 000000000000+, remote: 525251863cad |
|
120 | ancestor: 000000000000, local: 000000000000+, remote: 525251863cad | |
121 | lfs: assuming remote store: http://localhost:$HGPORT/subdir/mount/point/.git/info/lfs |
|
121 | lfs: assuming remote store: http://localhost:$HGPORT/subdir/mount/point/.git/info/lfs | |
122 | Status: 200 |
|
122 | Status: 200 | |
123 | Content-Length: 371 |
|
123 | Content-Length: 371 | |
124 | Content-Type: application/vnd.git-lfs+json |
|
124 | Content-Type: application/vnd.git-lfs+json | |
125 | Date: $HTTP_DATE$ |
|
125 | Date: $HTTP_DATE$ | |
126 | Server: testing stub value |
|
126 | Server: testing stub value | |
127 | { |
|
127 | { | |
128 | "objects": [ |
|
128 | "objects": [ | |
129 | { |
|
129 | { | |
130 | "actions": { |
|
130 | "actions": { | |
131 | "download": { |
|
131 | "download": { | |
132 | "expires_at": "$ISO_8601_DATE_TIME$" |
|
132 | "expires_at": "$ISO_8601_DATE_TIME$" | |
133 | "header": { |
|
133 | "header": { | |
134 | "Accept": "application/vnd.git-lfs" |
|
134 | "Accept": "application/vnd.git-lfs" | |
135 | } |
|
135 | } | |
136 | "href": "http://localhost:$HGPORT/subdir/mount/point/.hg/lfs/objects/f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e" |
|
136 | "href": "http://localhost:$HGPORT/subdir/mount/point/.hg/lfs/objects/f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e" | |
137 | } |
|
137 | } | |
138 | } |
|
138 | } | |
139 | "oid": "f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e" |
|
139 | "oid": "f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e" | |
140 | "size": 20 |
|
140 | "size": 20 | |
141 | } |
|
141 | } | |
142 | ] |
|
142 | ] | |
143 | "transfer": "basic" |
|
143 | "transfer": "basic" | |
144 | } |
|
144 | } | |
145 | lfs: downloading f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e (20 bytes) |
|
145 | lfs: downloading f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e (20 bytes) | |
146 | Status: 200 |
|
146 | Status: 200 | |
147 | Content-Length: 20 |
|
147 | Content-Length: 20 | |
148 | Content-Type: application/octet-stream |
|
148 | Content-Type: application/octet-stream | |
149 | Date: $HTTP_DATE$ |
|
149 | Date: $HTTP_DATE$ | |
150 | Server: testing stub value |
|
150 | Server: testing stub value | |
151 | lfs: adding f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e to the usercache |
|
151 | lfs: adding f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e to the usercache | |
152 | lfs: processed: f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e |
|
152 | lfs: processed: f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e | |
153 | lfs: downloaded 1 files (20 bytes) |
|
153 | lfs: downloaded 1 files (20 bytes) | |
154 | lfs.bin: remote created -> g |
|
154 | lfs.bin: remote created -> g | |
155 | getting lfs.bin |
|
155 | getting lfs.bin | |
156 | lfs: found f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e in the local lfs store |
|
156 | lfs: found f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e in the local lfs store | |
157 | 1 files updated, 0 files merged, 0 files removed, 0 files unresolved |
|
157 | 1 files updated, 0 files merged, 0 files removed, 0 files unresolved | |
158 | (sent 3 HTTP requests and * bytes; received * bytes in responses) (glob) |
|
158 | (sent 3 HTTP requests and * bytes; received * bytes in responses) (glob) | |
159 |
|
159 | |||
160 | $ "$PYTHON" $RUNTESTDIR/killdaemons.py $DAEMON_PIDS |
|
160 | $ "$PYTHON" $RUNTESTDIR/killdaemons.py $DAEMON_PIDS | |
161 |
|
161 | |||
162 | $ cat $TESTTMP/access.log $TESTTMP/errors.log |
|
162 | $ cat $TESTTMP/access.log $TESTTMP/errors.log | |
163 | $LOCALIP - - [$LOGDATE$] "POST /missing/objects/batch HTTP/1.1" 404 - (glob) |
|
163 | $LOCALIP - - [$LOGDATE$] "POST /missing/objects/batch HTTP/1.1" 404 - (glob) | |
164 | $LOCALIP - - [$LOGDATE$] "GET /subdir/mount/point?cmd=capabilities HTTP/1.1" 200 - (glob) |
|
164 | $LOCALIP - - [$LOGDATE$] "GET /subdir/mount/point?cmd=capabilities HTTP/1.1" 200 - (glob) | |
165 | $LOCALIP - - [$LOGDATE$] "GET /subdir/mount/point?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
165 | $LOCALIP - - [$LOGDATE$] "GET /subdir/mount/point?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
166 | $LOCALIP - - [$LOGDATE$] "GET /subdir/mount/point?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bookmarks=1&bundlecaps=HG20%2Cbundle2%3DHG20%250Abookmarks%250Achangegroup%253D01%252C02%252C03%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Aphases%253Dheads%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps%250Arev-branch-cache%250Astream%253Dv2&cg=1&common=0000000000000000000000000000000000000000&heads=525251863cad618e55d483555f3d00a2ca99597e&listkeys=bookmarks&phases=1 x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
166 | $LOCALIP - - [$LOGDATE$] "GET /subdir/mount/point?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bookmarks=1&bundlecaps=HG20%2Cbundle2%3DHG20%250Abookmarks%250Achangegroup%253D01%252C02%252C03%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Aphases%253Dheads%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps%250Arev-branch-cache%250Astream%253Dv2&cg=1&common=0000000000000000000000000000000000000000&heads=525251863cad618e55d483555f3d00a2ca99597e&listkeys=bookmarks&phases=1 x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
167 | $LOCALIP - - [$LOGDATE$] "POST /subdir/mount/point/.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob) |
|
167 | $LOCALIP - - [$LOGDATE$] "POST /subdir/mount/point/.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob) | |
168 | $LOCALIP - - [$LOGDATE$] "GET /subdir/mount/point/.hg/lfs/objects/f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e HTTP/1.1" 200 - (glob) |
|
168 | $LOCALIP - - [$LOGDATE$] "GET /subdir/mount/point/.hg/lfs/objects/f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e HTTP/1.1" 200 - (glob) | |
169 |
|
169 | |||
170 | Blobs that already exist in the usercache are linked into the repo store, even |
|
170 | Blobs that already exist in the usercache are linked into the repo store, even | |
171 | though the client doesn't send the blob. |
|
171 | though the client doesn't send the blob. | |
172 |
|
172 | |||
173 | $ hg init server2 |
|
173 | $ hg init server2 | |
174 | $ hg --config "lfs.usercache=$TESTTMP/servercache" -R server2 serve -d \ |
|
174 | $ hg --config "lfs.usercache=$TESTTMP/servercache" -R server2 serve -d \ | |
175 | > -p $HGPORT --pid-file=hg.pid \ |
|
175 | > -p $HGPORT --pid-file=hg.pid \ | |
176 | > -A $TESTTMP/access.log -E $TESTTMP/errors.log |
|
176 | > -A $TESTTMP/access.log -E $TESTTMP/errors.log | |
177 | $ cat hg.pid >> $DAEMON_PIDS |
|
177 | $ cat hg.pid >> $DAEMON_PIDS | |
178 |
|
178 | |||
179 | $ hg --config "lfs.usercache=$TESTTMP/servercache" -R cloned2 --debug \ |
|
179 | $ hg --config "lfs.usercache=$TESTTMP/servercache" -R cloned2 --debug \ | |
180 | > push http://localhost:$HGPORT | grep '^[{} ]' |
|
180 | > push http://localhost:$HGPORT | grep '^[{} ]' | |
181 | { |
|
181 | { | |
182 | "objects": [ |
|
182 | "objects": [ | |
183 | { |
|
183 | { | |
184 | "oid": "f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e" |
|
184 | "oid": "f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e" | |
185 | "size": 20 |
|
185 | "size": 20 | |
186 | } |
|
186 | } | |
187 | ] |
|
187 | ] | |
188 | "transfer": "basic" |
|
188 | "transfer": "basic" | |
189 | } |
|
189 | } | |
190 | $ find server2/.hg/store/lfs/objects | sort |
|
190 | $ find server2/.hg/store/lfs/objects | sort | |
191 | server2/.hg/store/lfs/objects |
|
191 | server2/.hg/store/lfs/objects | |
192 | server2/.hg/store/lfs/objects/f0 |
|
192 | server2/.hg/store/lfs/objects/f0 | |
193 | server2/.hg/store/lfs/objects/f0/3217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e |
|
193 | server2/.hg/store/lfs/objects/f0/3217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e | |
194 | $ "$PYTHON" $RUNTESTDIR/killdaemons.py $DAEMON_PIDS |
|
194 | $ "$PYTHON" $RUNTESTDIR/killdaemons.py $DAEMON_PIDS | |
195 | $ cat $TESTTMP/errors.log |
|
195 | $ cat $TESTTMP/errors.log | |
196 |
|
196 | |||
197 | $ cat >> $TESTTMP/lfsstoreerror.py <<EOF |
|
197 | $ cat >> $TESTTMP/lfsstoreerror.py <<EOF | |
198 | > import errno |
|
198 | > import errno | |
199 | > from hgext.lfs import blobstore |
|
199 | > from hgext.lfs import blobstore | |
200 | > |
|
200 | > | |
201 | > _numverifies = 0 |
|
201 | > _numverifies = 0 | |
202 | > _readerr = True |
|
202 | > _readerr = True | |
203 | > |
|
203 | > | |
204 | > def reposetup(ui, repo): |
|
204 | > def reposetup(ui, repo): | |
205 | > # Nothing to do with a remote repo |
|
205 | > # Nothing to do with a remote repo | |
206 | > if not repo.local(): |
|
206 | > if not repo.local(): | |
207 | > return |
|
207 | > return | |
208 | > |
|
208 | > | |
209 | > store = repo.svfs.lfslocalblobstore |
|
209 | > store = repo.svfs.lfslocalblobstore | |
210 | > class badstore(store.__class__): |
|
210 | > class badstore(store.__class__): | |
211 | > def download(self, oid, src): |
|
211 | > def download(self, oid, src): | |
212 | > '''Called in the server to handle reading from the client in a |
|
212 | > '''Called in the server to handle reading from the client in a | |
213 | > PUT request.''' |
|
213 | > PUT request.''' | |
214 | > origread = src.read |
|
214 | > origread = src.read | |
215 | > def _badread(nbytes): |
|
215 | > def _badread(nbytes): | |
216 | > # Simulate bad data/checksum failure from the client |
|
216 | > # Simulate bad data/checksum failure from the client | |
217 | > return b'0' * len(origread(nbytes)) |
|
217 | > return b'0' * len(origread(nbytes)) | |
218 | > src.read = _badread |
|
218 | > src.read = _badread | |
219 | > super(badstore, self).download(oid, src) |
|
219 | > super(badstore, self).download(oid, src) | |
220 | > |
|
220 | > | |
221 | > def _read(self, vfs, oid, verify): |
|
221 | > def _read(self, vfs, oid, verify): | |
222 | > '''Called in the server to read data for a GET request, and then |
|
222 | > '''Called in the server to read data for a GET request, and then | |
223 | > calls self._verify() on it before returning.''' |
|
223 | > calls self._verify() on it before returning.''' | |
224 | > global _readerr |
|
224 | > global _readerr | |
225 | > # One time simulation of a read error |
|
225 | > # One time simulation of a read error | |
226 | > if _readerr: |
|
226 | > if _readerr: | |
227 | > _readerr = False |
|
227 | > _readerr = False | |
228 | > raise IOError(errno.EIO, '%s: I/O error' % oid) |
|
228 | > raise IOError(errno.EIO, '%s: I/O error' % oid) | |
229 | > # Simulate corrupt content on client download |
|
229 | > # Simulate corrupt content on client download | |
230 | > blobstore._verify(oid, 'dummy content') |
|
230 | > blobstore._verify(oid, 'dummy content') | |
231 | > |
|
231 | > | |
232 | > def verify(self, oid): |
|
232 | > def verify(self, oid): | |
233 | > '''Called in the server to populate the Batch API response, |
|
233 | > '''Called in the server to populate the Batch API response, | |
234 | > letting the client re-upload if the file is corrupt.''' |
|
234 | > letting the client re-upload if the file is corrupt.''' | |
235 | > # Fail verify in Batch API for one clone command and one push |
|
235 | > # Fail verify in Batch API for one clone command and one push | |
236 | > # command with an IOError. Then let it through to access other |
|
236 | > # command with an IOError. Then let it through to access other | |
237 | > # functions. Checksum failure is tested elsewhere. |
|
237 | > # functions. Checksum failure is tested elsewhere. | |
238 | > global _numverifies |
|
238 | > global _numverifies | |
239 | > _numverifies += 1 |
|
239 | > _numverifies += 1 | |
240 | > if _numverifies <= 2: |
|
240 | > if _numverifies <= 2: | |
241 | > raise IOError(errno.EIO, '%s: I/O error' % oid) |
|
241 | > raise IOError(errno.EIO, '%s: I/O error' % oid) | |
242 | > return super(badstore, self).verify(oid) |
|
242 | > return super(badstore, self).verify(oid) | |
243 | > |
|
243 | > | |
244 | > store.__class__ = badstore |
|
244 | > store.__class__ = badstore | |
245 | > EOF |
|
245 | > EOF | |
246 |
|
246 | |||
247 | $ rm -rf `hg config lfs.usercache` |
|
247 | $ rm -rf `hg config lfs.usercache` | |
248 | $ rm -f $TESTTMP/access.log $TESTTMP/errors.log |
|
248 | $ rm -f $TESTTMP/access.log $TESTTMP/errors.log | |
249 | $ hg --config "lfs.usercache=$TESTTMP/servercache" \ |
|
249 | $ hg --config "lfs.usercache=$TESTTMP/servercache" \ | |
250 | > --config extensions.lfsstoreerror=$TESTTMP/lfsstoreerror.py \ |
|
250 | > --config extensions.lfsstoreerror=$TESTTMP/lfsstoreerror.py \ | |
251 | > -R server serve -d \ |
|
251 | > -R server serve -d \ | |
252 | > -p $HGPORT1 --pid-file=hg.pid -A $TESTTMP/access.log -E $TESTTMP/errors.log |
|
252 | > -p $HGPORT1 --pid-file=hg.pid -A $TESTTMP/access.log -E $TESTTMP/errors.log | |
253 | $ cat hg.pid >> $DAEMON_PIDS |
|
253 | $ cat hg.pid >> $DAEMON_PIDS | |
254 |
|
254 | |||
255 | Test an I/O error in localstore.verify() (Batch API) with GET |
|
255 | Test an I/O error in localstore.verify() (Batch API) with GET | |
256 |
|
256 | |||
257 | $ hg clone http://localhost:$HGPORT1 httpclone2 |
|
257 | $ hg clone http://localhost:$HGPORT1 httpclone2 | |
258 | (remote is using large file support (lfs); lfs will be enabled for this repository) |
|
258 | (remote is using large file support (lfs); lfs will be enabled for this repository) | |
259 | requesting all changes |
|
259 | requesting all changes | |
260 | adding changesets |
|
260 | adding changesets | |
261 | adding manifests |
|
261 | adding manifests | |
262 | adding file changes |
|
262 | adding file changes | |
263 | added 1 changesets with 1 changes to 1 files |
|
263 | added 1 changesets with 1 changes to 1 files | |
264 | new changesets 525251863cad |
|
264 | new changesets 525251863cad | |
265 | updating to branch default |
|
265 | updating to branch default | |
266 | abort: LFS server error for "lfs.bin": Internal server error! |
|
266 | abort: LFS server error for "lfs.bin": Internal server error! | |
267 | [255] |
|
267 | [255] | |
268 |
|
268 | |||
269 | Test an I/O error in localstore.verify() (Batch API) with PUT |
|
269 | Test an I/O error in localstore.verify() (Batch API) with PUT | |
270 |
|
270 | |||
271 | $ echo foo > client/lfs.bin |
|
271 | $ echo foo > client/lfs.bin | |
272 | $ hg -R client ci -m 'mod lfs' |
|
272 | $ hg -R client ci -m 'mod lfs' | |
273 | $ hg -R client push http://localhost:$HGPORT1 |
|
273 | $ hg -R client push http://localhost:$HGPORT1 | |
274 | pushing to http://localhost:$HGPORT1/ |
|
274 | pushing to http://localhost:$HGPORT1/ | |
275 | searching for changes |
|
275 | searching for changes | |
276 | abort: LFS server error for "unknown": Internal server error! |
|
276 | abort: LFS server error for "unknown": Internal server error! | |
277 | [255] |
|
277 | [255] | |
278 | TODO: figure out how to associate the file name in the error above |
|
278 | TODO: figure out how to associate the file name in the error above | |
279 |
|
279 | |||
280 | Test a bad checksum sent by the client in the transfer API |
|
280 | Test a bad checksum sent by the client in the transfer API | |
281 |
|
281 | |||
282 | $ hg -R client push http://localhost:$HGPORT1 |
|
282 | $ hg -R client push http://localhost:$HGPORT1 | |
283 | pushing to http://localhost:$HGPORT1/ |
|
283 | pushing to http://localhost:$HGPORT1/ | |
284 | searching for changes |
|
284 | searching for changes | |
285 | abort: HTTP error: HTTP Error 422: corrupt blob (oid=b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c, action=upload)! |
|
285 | abort: LFS HTTP error: HTTP Error 422: corrupt blob (oid=b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c, action=upload)! | |
286 | [255] |
|
286 | [255] | |
287 |
|
287 | |||
288 | $ echo 'test lfs file' > server/lfs3.bin |
|
288 | $ echo 'test lfs file' > server/lfs3.bin | |
289 | $ hg --config experimental.lfs.disableusercache=True \ |
|
289 | $ hg --config experimental.lfs.disableusercache=True \ | |
290 | > -R server ci -Aqm 'another lfs file' |
|
290 | > -R server ci -Aqm 'another lfs file' | |
291 | $ hg -R client pull -q http://localhost:$HGPORT1 |
|
291 | $ hg -R client pull -q http://localhost:$HGPORT1 | |
292 |
|
292 | |||
293 | Test an I/O error during the processing of the GET request |
|
293 | Test an I/O error during the processing of the GET request | |
294 |
|
294 | |||
295 | $ hg --config lfs.url=http://localhost:$HGPORT1/.git/info/lfs \ |
|
295 | $ hg --config lfs.url=http://localhost:$HGPORT1/.git/info/lfs \ | |
296 | > -R client update -r tip |
|
296 | > -R client update -r tip | |
297 | abort: HTTP error: HTTP Error 500: Internal Server Error (oid=276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d, action=download)! |
|
297 | abort: LFS HTTP error: HTTP Error 500: Internal Server Error (oid=276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d, action=download)! | |
298 | [255] |
|
298 | [255] | |
299 |
|
299 | |||
300 | Test a checksum failure during the processing of the GET request |
|
300 | Test a checksum failure during the processing of the GET request | |
301 |
|
301 | |||
302 | $ hg --config lfs.url=http://localhost:$HGPORT1/.git/info/lfs \ |
|
302 | $ hg --config lfs.url=http://localhost:$HGPORT1/.git/info/lfs \ | |
303 | > -R client update -r tip |
|
303 | > -R client update -r tip | |
304 | abort: HTTP error: HTTP Error 422: corrupt blob (oid=276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d, action=download)! |
|
304 | abort: LFS HTTP error: HTTP Error 422: corrupt blob (oid=276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d, action=download)! | |
305 | [255] |
|
305 | [255] | |
306 |
|
306 | |||
307 | $ "$PYTHON" $RUNTESTDIR/killdaemons.py $DAEMON_PIDS |
|
307 | $ "$PYTHON" $RUNTESTDIR/killdaemons.py $DAEMON_PIDS | |
308 |
|
308 | |||
309 | $ cat $TESTTMP/access.log |
|
309 | $ cat $TESTTMP/access.log | |
310 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) |
|
310 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) | |
311 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
311 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
312 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bookmarks=1&bundlecaps=HG20%2Cbundle2%3DHG20%250Abookmarks%250Achangegroup%253D01%252C02%252C03%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Aphases%253Dheads%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps%250Arev-branch-cache%250Astream%253Dv2&cg=1&common=0000000000000000000000000000000000000000&heads=525251863cad618e55d483555f3d00a2ca99597e&listkeys=bookmarks&phases=1 x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
312 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bookmarks=1&bundlecaps=HG20%2Cbundle2%3DHG20%250Abookmarks%250Achangegroup%253D01%252C02%252C03%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Aphases%253Dheads%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps%250Arev-branch-cache%250Astream%253Dv2&cg=1&common=0000000000000000000000000000000000000000&heads=525251863cad618e55d483555f3d00a2ca99597e&listkeys=bookmarks&phases=1 x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
313 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob) |
|
313 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob) | |
314 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) |
|
314 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) | |
315 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D392c05922088bacf8e68a6939b480017afbf245d x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
315 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D392c05922088bacf8e68a6939b480017afbf245d x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
316 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
316 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
317 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
317 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
318 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=branchmap HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
318 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=branchmap HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
319 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
319 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
320 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob) |
|
320 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob) | |
321 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) |
|
321 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) | |
322 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D392c05922088bacf8e68a6939b480017afbf245d x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
322 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D392c05922088bacf8e68a6939b480017afbf245d x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
323 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
323 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
324 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
324 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
325 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=branchmap HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
325 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=branchmap HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
326 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
326 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
327 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob) |
|
327 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob) | |
328 | $LOCALIP - - [$LOGDATE$] "PUT /.hg/lfs/objects/b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c HTTP/1.1" 422 - (glob) |
|
328 | $LOCALIP - - [$LOGDATE$] "PUT /.hg/lfs/objects/b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c HTTP/1.1" 422 - (glob) | |
329 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) |
|
329 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) | |
330 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D392c05922088bacf8e68a6939b480017afbf245d x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
330 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D392c05922088bacf8e68a6939b480017afbf245d x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
331 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bookmarks=1&bundlecaps=HG20%2Cbundle2%3DHG20%250Abookmarks%250Achangegroup%253D01%252C02%252C03%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Aphases%253Dheads%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps%250Arev-branch-cache%250Astream%253Dv2&cg=1&common=525251863cad618e55d483555f3d00a2ca99597e&heads=506bf3d83f78c54b89e81c6411adee19fdf02156+525251863cad618e55d483555f3d00a2ca99597e&listkeys=bookmarks&phases=1 x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
331 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bookmarks=1&bundlecaps=HG20%2Cbundle2%3DHG20%250Abookmarks%250Achangegroup%253D01%252C02%252C03%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Aphases%253Dheads%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps%250Arev-branch-cache%250Astream%253Dv2&cg=1&common=525251863cad618e55d483555f3d00a2ca99597e&heads=506bf3d83f78c54b89e81c6411adee19fdf02156+525251863cad618e55d483555f3d00a2ca99597e&listkeys=bookmarks&phases=1 x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
332 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob) |
|
332 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob) | |
333 | $LOCALIP - - [$LOGDATE$] "GET /.hg/lfs/objects/276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d HTTP/1.1" 500 - (glob) |
|
333 | $LOCALIP - - [$LOGDATE$] "GET /.hg/lfs/objects/276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d HTTP/1.1" 500 - (glob) | |
334 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob) |
|
334 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob) | |
335 | $LOCALIP - - [$LOGDATE$] "GET /.hg/lfs/objects/276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d HTTP/1.1" 422 - (glob) |
|
335 | $LOCALIP - - [$LOGDATE$] "GET /.hg/lfs/objects/276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d HTTP/1.1" 422 - (glob) | |
336 |
|
336 | |||
337 | $ grep -v ' File "' $TESTTMP/errors.log |
|
337 | $ grep -v ' File "' $TESTTMP/errors.log | |
338 | $LOCALIP - - [$ERRDATE$] HG error: Exception happened while processing request '/.git/info/lfs/objects/batch': (glob) |
|
338 | $LOCALIP - - [$ERRDATE$] HG error: Exception happened while processing request '/.git/info/lfs/objects/batch': (glob) | |
339 | $LOCALIP - - [$ERRDATE$] HG error: Traceback (most recent call last): (glob) |
|
339 | $LOCALIP - - [$ERRDATE$] HG error: Traceback (most recent call last): (glob) | |
340 | $LOCALIP - - [$ERRDATE$] HG error: verifies = store.verify(oid) (glob) |
|
340 | $LOCALIP - - [$ERRDATE$] HG error: verifies = store.verify(oid) (glob) | |
341 | $LOCALIP - - [$ERRDATE$] HG error: raise IOError(errno.EIO, '%s: I/O error' % oid) (glob) |
|
341 | $LOCALIP - - [$ERRDATE$] HG error: raise IOError(errno.EIO, '%s: I/O error' % oid) (glob) | |
342 | $LOCALIP - - [$ERRDATE$] HG error: IOError: [Errno 5] f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e: I/O error (glob) |
|
342 | $LOCALIP - - [$ERRDATE$] HG error: IOError: [Errno 5] f03217a32529a28a42d03b1244fe09b6e0f9fd06d7b966d4d50567be2abe6c0e: I/O error (glob) | |
343 | $LOCALIP - - [$ERRDATE$] HG error: (glob) |
|
343 | $LOCALIP - - [$ERRDATE$] HG error: (glob) | |
344 | $LOCALIP - - [$ERRDATE$] HG error: Exception happened while processing request '/.git/info/lfs/objects/batch': (glob) |
|
344 | $LOCALIP - - [$ERRDATE$] HG error: Exception happened while processing request '/.git/info/lfs/objects/batch': (glob) | |
345 | $LOCALIP - - [$ERRDATE$] HG error: Traceback (most recent call last): (glob) |
|
345 | $LOCALIP - - [$ERRDATE$] HG error: Traceback (most recent call last): (glob) | |
346 | $LOCALIP - - [$ERRDATE$] HG error: verifies = store.verify(oid) (glob) |
|
346 | $LOCALIP - - [$ERRDATE$] HG error: verifies = store.verify(oid) (glob) | |
347 | $LOCALIP - - [$ERRDATE$] HG error: raise IOError(errno.EIO, '%s: I/O error' % oid) (glob) |
|
347 | $LOCALIP - - [$ERRDATE$] HG error: raise IOError(errno.EIO, '%s: I/O error' % oid) (glob) | |
348 | $LOCALIP - - [$ERRDATE$] HG error: IOError: [Errno 5] b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c: I/O error (glob) |
|
348 | $LOCALIP - - [$ERRDATE$] HG error: IOError: [Errno 5] b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c: I/O error (glob) | |
349 | $LOCALIP - - [$ERRDATE$] HG error: (glob) |
|
349 | $LOCALIP - - [$ERRDATE$] HG error: (glob) | |
350 | $LOCALIP - - [$ERRDATE$] HG error: Exception happened while processing request '/.hg/lfs/objects/b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c': (glob) |
|
350 | $LOCALIP - - [$ERRDATE$] HG error: Exception happened while processing request '/.hg/lfs/objects/b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c': (glob) | |
351 | $LOCALIP - - [$ERRDATE$] HG error: Traceback (most recent call last): (glob) |
|
351 | $LOCALIP - - [$ERRDATE$] HG error: Traceback (most recent call last): (glob) | |
352 | $LOCALIP - - [$ERRDATE$] HG error: localstore.download(oid, req.bodyfh) (glob) |
|
352 | $LOCALIP - - [$ERRDATE$] HG error: localstore.download(oid, req.bodyfh) (glob) | |
353 | $LOCALIP - - [$ERRDATE$] HG error: super(badstore, self).download(oid, src) (glob) |
|
353 | $LOCALIP - - [$ERRDATE$] HG error: super(badstore, self).download(oid, src) (glob) | |
354 | $LOCALIP - - [$ERRDATE$] HG error: % oid) (glob) |
|
354 | $LOCALIP - - [$ERRDATE$] HG error: % oid) (glob) | |
355 | $LOCALIP - - [$ERRDATE$] HG error: LfsCorruptionError: corrupt remote lfs object: b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c (glob) |
|
355 | $LOCALIP - - [$ERRDATE$] HG error: LfsCorruptionError: corrupt remote lfs object: b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c (glob) | |
356 | $LOCALIP - - [$ERRDATE$] HG error: (glob) |
|
356 | $LOCALIP - - [$ERRDATE$] HG error: (glob) | |
357 | $LOCALIP - - [$ERRDATE$] Exception happened during processing request '/.hg/lfs/objects/276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d': (glob) |
|
357 | $LOCALIP - - [$ERRDATE$] Exception happened during processing request '/.hg/lfs/objects/276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d': (glob) | |
358 | Traceback (most recent call last): |
|
358 | Traceback (most recent call last): | |
359 | self.do_write() |
|
359 | self.do_write() | |
360 | self.do_hgweb() |
|
360 | self.do_hgweb() | |
361 | for chunk in self.server.application(env, self._start_response): |
|
361 | for chunk in self.server.application(env, self._start_response): | |
362 | for r in self._runwsgi(req, res, repo): |
|
362 | for r in self._runwsgi(req, res, repo): | |
363 | rctx, req, res, self.check_perm) |
|
363 | rctx, req, res, self.check_perm) | |
364 | return func(*(args + a), **kw) |
|
364 | return func(*(args + a), **kw) | |
365 | lambda perm: |
|
365 | lambda perm: | |
366 | res.setbodybytes(localstore.read(oid)) |
|
366 | res.setbodybytes(localstore.read(oid)) | |
367 | blob = self._read(self.vfs, oid, verify) |
|
367 | blob = self._read(self.vfs, oid, verify) | |
368 | raise IOError(errno.EIO, '%s: I/O error' % oid) |
|
368 | raise IOError(errno.EIO, '%s: I/O error' % oid) | |
369 | IOError: [Errno 5] 276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d: I/O error |
|
369 | IOError: [Errno 5] 276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d: I/O error | |
370 |
|
370 | |||
371 | $LOCALIP - - [$ERRDATE$] HG error: Exception happened while processing request '/.hg/lfs/objects/276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d': (glob) |
|
371 | $LOCALIP - - [$ERRDATE$] HG error: Exception happened while processing request '/.hg/lfs/objects/276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d': (glob) | |
372 | $LOCALIP - - [$ERRDATE$] HG error: Traceback (most recent call last): (glob) |
|
372 | $LOCALIP - - [$ERRDATE$] HG error: Traceback (most recent call last): (glob) | |
373 | $LOCALIP - - [$ERRDATE$] HG error: res.setbodybytes(localstore.read(oid)) (glob) |
|
373 | $LOCALIP - - [$ERRDATE$] HG error: res.setbodybytes(localstore.read(oid)) (glob) | |
374 | $LOCALIP - - [$ERRDATE$] HG error: blob = self._read(self.vfs, oid, verify) (glob) |
|
374 | $LOCALIP - - [$ERRDATE$] HG error: blob = self._read(self.vfs, oid, verify) (glob) | |
375 | $LOCALIP - - [$ERRDATE$] HG error: blobstore._verify(oid, 'dummy content') (glob) |
|
375 | $LOCALIP - - [$ERRDATE$] HG error: blobstore._verify(oid, 'dummy content') (glob) | |
376 | $LOCALIP - - [$ERRDATE$] HG error: hint=_('run hg verify')) (glob) |
|
376 | $LOCALIP - - [$ERRDATE$] HG error: hint=_('run hg verify')) (glob) | |
377 | $LOCALIP - - [$ERRDATE$] HG error: LfsCorruptionError: detected corrupt lfs object: 276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d (glob) |
|
377 | $LOCALIP - - [$ERRDATE$] HG error: LfsCorruptionError: detected corrupt lfs object: 276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d (glob) | |
378 | $LOCALIP - - [$ERRDATE$] HG error: (glob) |
|
378 | $LOCALIP - - [$ERRDATE$] HG error: (glob) | |
379 |
|
379 | |||
380 | Basic Authorization headers are returned by the Batch API, and sent back with |
|
380 | Basic Authorization headers are returned by the Batch API, and sent back with | |
381 | the GET/PUT request. |
|
381 | the GET/PUT request. | |
382 |
|
382 | |||
383 | $ rm -f $TESTTMP/access.log $TESTTMP/errors.log |
|
383 | $ rm -f $TESTTMP/access.log $TESTTMP/errors.log | |
384 |
|
384 | |||
385 | $ cat >> $HGRCPATH << EOF |
|
385 | $ cat >> $HGRCPATH << EOF | |
386 | > [experimental] |
|
386 | > [experimental] | |
387 | > lfs.disableusercache = True |
|
387 | > lfs.disableusercache = True | |
388 | > [auth] |
|
388 | > [auth] | |
389 | > l.schemes=http |
|
389 | > l.schemes=http | |
390 | > l.prefix=lo |
|
390 | > l.prefix=lo | |
391 | > l.username=user |
|
391 | > l.username=user | |
392 | > l.password=pass |
|
392 | > l.password=pass | |
393 | > EOF |
|
393 | > EOF | |
394 |
|
394 | |||
395 | $ cat << EOF > userpass.py |
|
395 | $ cat << EOF > userpass.py | |
396 | > import base64 |
|
396 | > import base64 | |
397 | > from mercurial.hgweb import common |
|
397 | > from mercurial.hgweb import common | |
398 | > def perform_authentication(hgweb, req, op): |
|
398 | > def perform_authentication(hgweb, req, op): | |
399 | > auth = req.headers.get(b'Authorization') |
|
399 | > auth = req.headers.get(b'Authorization') | |
400 | > if not auth: |
|
400 | > if not auth: | |
401 | > raise common.ErrorResponse(common.HTTP_UNAUTHORIZED, b'who', |
|
401 | > raise common.ErrorResponse(common.HTTP_UNAUTHORIZED, b'who', | |
402 | > [(b'WWW-Authenticate', b'Basic Realm="mercurial"')]) |
|
402 | > [(b'WWW-Authenticate', b'Basic Realm="mercurial"')]) | |
403 | > if base64.b64decode(auth.split()[1]).split(b':', 1) != [b'user', |
|
403 | > if base64.b64decode(auth.split()[1]).split(b':', 1) != [b'user', | |
404 | > b'pass']: |
|
404 | > b'pass']: | |
405 | > raise common.ErrorResponse(common.HTTP_FORBIDDEN, b'no') |
|
405 | > raise common.ErrorResponse(common.HTTP_FORBIDDEN, b'no') | |
406 | > def extsetup(): |
|
406 | > def extsetup(): | |
407 | > common.permhooks.insert(0, perform_authentication) |
|
407 | > common.permhooks.insert(0, perform_authentication) | |
408 | > EOF |
|
408 | > EOF | |
409 |
|
409 | |||
410 | $ hg --config extensions.x=$TESTTMP/userpass.py \ |
|
410 | $ hg --config extensions.x=$TESTTMP/userpass.py \ | |
411 | > -R server serve -d -p $HGPORT1 --pid-file=hg.pid \ |
|
411 | > -R server serve -d -p $HGPORT1 --pid-file=hg.pid \ | |
412 | > -A $TESTTMP/access.log -E $TESTTMP/errors.log |
|
412 | > -A $TESTTMP/access.log -E $TESTTMP/errors.log | |
413 | $ mv hg.pid $DAEMON_PIDS |
|
413 | $ mv hg.pid $DAEMON_PIDS | |
414 |
|
414 | |||
415 | $ hg clone --debug http://localhost:$HGPORT1 auth_clone | egrep '^[{}]| ' |
|
415 | $ hg clone --debug http://localhost:$HGPORT1 auth_clone | egrep '^[{}]| ' | |
416 | { |
|
416 | { | |
417 | "objects": [ |
|
417 | "objects": [ | |
418 | { |
|
418 | { | |
419 | "actions": { |
|
419 | "actions": { | |
420 | "download": { |
|
420 | "download": { | |
421 | "expires_at": "$ISO_8601_DATE_TIME$" |
|
421 | "expires_at": "$ISO_8601_DATE_TIME$" | |
422 | "header": { |
|
422 | "header": { | |
423 | "Accept": "application/vnd.git-lfs" |
|
423 | "Accept": "application/vnd.git-lfs" | |
424 | "Authorization": "Basic dXNlcjpwYXNz" |
|
424 | "Authorization": "Basic dXNlcjpwYXNz" | |
425 | } |
|
425 | } | |
426 | "href": "http://localhost:$HGPORT1/.hg/lfs/objects/276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d" |
|
426 | "href": "http://localhost:$HGPORT1/.hg/lfs/objects/276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d" | |
427 | } |
|
427 | } | |
428 | } |
|
428 | } | |
429 | "oid": "276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d" |
|
429 | "oid": "276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d" | |
430 | "size": 14 |
|
430 | "size": 14 | |
431 | } |
|
431 | } | |
432 | ] |
|
432 | ] | |
433 | "transfer": "basic" |
|
433 | "transfer": "basic" | |
434 | } |
|
434 | } | |
435 |
|
435 | |||
436 | $ echo 'another blob' > auth_clone/lfs.blob |
|
436 | $ echo 'another blob' > auth_clone/lfs.blob | |
437 | $ hg -R auth_clone ci -Aqm 'add blob' |
|
437 | $ hg -R auth_clone ci -Aqm 'add blob' | |
438 | $ hg -R auth_clone --debug push | egrep '^[{}]| ' |
|
438 | $ hg -R auth_clone --debug push | egrep '^[{}]| ' | |
439 | { |
|
439 | { | |
440 | "objects": [ |
|
440 | "objects": [ | |
441 | { |
|
441 | { | |
442 | "actions": { |
|
442 | "actions": { | |
443 | "upload": { |
|
443 | "upload": { | |
444 | "expires_at": "$ISO_8601_DATE_TIME$" |
|
444 | "expires_at": "$ISO_8601_DATE_TIME$" | |
445 | "header": { |
|
445 | "header": { | |
446 | "Accept": "application/vnd.git-lfs" |
|
446 | "Accept": "application/vnd.git-lfs" | |
447 | "Authorization": "Basic dXNlcjpwYXNz" |
|
447 | "Authorization": "Basic dXNlcjpwYXNz" | |
448 | } |
|
448 | } | |
449 | "href": "http://localhost:$HGPORT1/.hg/lfs/objects/df14287d8d75f076a6459e7a3703ca583ca9fb3f4918caed10c77ac8622d49b3" |
|
449 | "href": "http://localhost:$HGPORT1/.hg/lfs/objects/df14287d8d75f076a6459e7a3703ca583ca9fb3f4918caed10c77ac8622d49b3" | |
450 | } |
|
450 | } | |
451 | } |
|
451 | } | |
452 | "oid": "df14287d8d75f076a6459e7a3703ca583ca9fb3f4918caed10c77ac8622d49b3" |
|
452 | "oid": "df14287d8d75f076a6459e7a3703ca583ca9fb3f4918caed10c77ac8622d49b3" | |
453 | "size": 13 |
|
453 | "size": 13 | |
454 | } |
|
454 | } | |
455 | ] |
|
455 | ] | |
456 | "transfer": "basic" |
|
456 | "transfer": "basic" | |
457 | } |
|
457 | } | |
458 |
|
458 | |||
459 | $ "$PYTHON" $RUNTESTDIR/killdaemons.py $DAEMON_PIDS |
|
459 | $ "$PYTHON" $RUNTESTDIR/killdaemons.py $DAEMON_PIDS | |
460 |
|
460 | |||
461 | $ cat $TESTTMP/access.log $TESTTMP/errors.log |
|
461 | $ cat $TESTTMP/access.log $TESTTMP/errors.log | |
462 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 401 - (glob) |
|
462 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 401 - (glob) | |
463 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) |
|
463 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) | |
464 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
464 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
465 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bookmarks=1&bundlecaps=HG20%2Cbundle2%3DHG20%250Abookmarks%250Achangegroup%253D01%252C02%252C03%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Aphases%253Dheads%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps%250Arev-branch-cache%250Astream%253Dv2&cg=1&common=0000000000000000000000000000000000000000&heads=506bf3d83f78c54b89e81c6411adee19fdf02156+525251863cad618e55d483555f3d00a2ca99597e&listkeys=bookmarks&phases=1 x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
465 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bookmarks=1&bundlecaps=HG20%2Cbundle2%3DHG20%250Abookmarks%250Achangegroup%253D01%252C02%252C03%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Aphases%253Dheads%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps%250Arev-branch-cache%250Astream%253Dv2&cg=1&common=0000000000000000000000000000000000000000&heads=506bf3d83f78c54b89e81c6411adee19fdf02156+525251863cad618e55d483555f3d00a2ca99597e&listkeys=bookmarks&phases=1 x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
466 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 401 - (glob) |
|
466 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 401 - (glob) | |
467 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob) |
|
467 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob) | |
468 | $LOCALIP - - [$LOGDATE$] "GET /.hg/lfs/objects/276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d HTTP/1.1" 200 - (glob) |
|
468 | $LOCALIP - - [$LOGDATE$] "GET /.hg/lfs/objects/276f73cfd75f9fb519810df5f5d96d6594ca2521abd86cbcd92122f7d51a1f3d HTTP/1.1" 200 - (glob) | |
469 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 401 - (glob) |
|
469 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 401 - (glob) | |
470 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) |
|
470 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) | |
471 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D525251863cad618e55d483555f3d00a2ca99597e+4d9397055dc0c205f3132f331f36353ab1a525a3 x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
471 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D525251863cad618e55d483555f3d00a2ca99597e+4d9397055dc0c205f3132f331f36353ab1a525a3 x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
472 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
472 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
473 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
473 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
474 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=branchmap HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
474 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=branchmap HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
475 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
475 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
476 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 401 - (glob) |
|
476 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 401 - (glob) | |
477 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob) |
|
477 | $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob) | |
478 | $LOCALIP - - [$LOGDATE$] "PUT /.hg/lfs/objects/df14287d8d75f076a6459e7a3703ca583ca9fb3f4918caed10c77ac8622d49b3 HTTP/1.1" 201 - (glob) |
|
478 | $LOCALIP - - [$LOGDATE$] "PUT /.hg/lfs/objects/df14287d8d75f076a6459e7a3703ca583ca9fb3f4918caed10c77ac8622d49b3 HTTP/1.1" 201 - (glob) | |
479 | $LOCALIP - - [$LOGDATE$] "POST /?cmd=unbundle HTTP/1.1" 200 - x-hgarg-1:heads=666f726365 x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
479 | $LOCALIP - - [$LOGDATE$] "POST /?cmd=unbundle HTTP/1.1" 200 - x-hgarg-1:heads=666f726365 x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) | |
480 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
|
480 | $LOCALIP - - [$LOGDATE$] "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=$USUAL_COMPRESSIONS$ partial-pull (glob) |
General Comments 0
You need to be logged in to leave comments.
Login now