##// END OF EJS Templates
largefiles: respect store.createmode in basestore.get...
Martin Geisler -
r16154:9b072a5f stable
parent child Browse files
Show More
@@ -1,205 +1,195 b''
1 1 # Copyright 2009-2010 Gregory P. Ward
2 2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
3 3 # Copyright 2010-2011 Fog Creek Software
4 4 # Copyright 2010-2011 Unity Technologies
5 5 #
6 6 # This software may be used and distributed according to the terms of the
7 7 # GNU General Public License version 2 or any later version.
8 8
9 9 '''base class for store implementations and store-related utility code'''
10 10
11 import os
12 import tempfile
13 11 import binascii
14 12 import re
15 13
16 14 from mercurial import util, node, hg
17 15 from mercurial.i18n import _
18 16
19 17 import lfutil
20 18
21 19 class StoreError(Exception):
22 20 '''Raised when there is a problem getting files from or putting
23 21 files to a central store.'''
24 22 def __init__(self, filename, hash, url, detail):
25 23 self.filename = filename
26 24 self.hash = hash
27 25 self.url = url
28 26 self.detail = detail
29 27
30 28 def longmessage(self):
31 29 if self.url:
32 30 return ('%s: %s\n'
33 31 '(failed URL: %s)\n'
34 32 % (self.filename, self.detail, self.url))
35 33 else:
36 34 return ('%s: %s\n'
37 35 '(no default or default-push path set in hgrc)\n'
38 36 % (self.filename, self.detail))
39 37
40 38 def __str__(self):
41 39 return "%s: %s" % (self.url, self.detail)
42 40
43 41 class basestore(object):
44 42 def __init__(self, ui, repo, url):
45 43 self.ui = ui
46 44 self.repo = repo
47 45 self.url = url
48 46
49 47 def put(self, source, hash):
50 48 '''Put source file into the store under <filename>/<hash>.'''
51 49 raise NotImplementedError('abstract method')
52 50
53 51 def exists(self, hash):
54 52 '''Check to see if the store contains the given hash.'''
55 53 raise NotImplementedError('abstract method')
56 54
57 55 def get(self, files):
58 56 '''Get the specified largefiles from the store and write to local
59 57 files under repo.root. files is a list of (filename, hash)
60 58 tuples. Return (success, missing), lists of files successfuly
61 59 downloaded and those not found in the store. success is a list
62 60 of (filename, hash) tuples; missing is a list of filenames that
63 61 we could not get. (The detailed error message will already have
64 62 been presented to the user, so missing is just supplied as a
65 63 summary.)'''
66 64 success = []
67 65 missing = []
68 66 ui = self.ui
69 67
70 68 at = 0
71 69 for filename, hash in files:
72 70 ui.progress(_('getting largefiles'), at, unit='lfile',
73 71 total=len(files))
74 72 at += 1
75 73 ui.note(_('getting %s:%s\n') % (filename, hash))
76 74
77 75 storefilename = lfutil.storepath(self.repo, hash)
78 storedir = os.path.dirname(storefilename)
79
80 # No need to pass mode='wb' to fdopen(), since mkstemp() already
81 # opened the file in binary mode.
82 (tmpfd, tmpfilename) = tempfile.mkstemp(
83 dir=storedir, prefix=os.path.basename(filename))
84 tmpfile = os.fdopen(tmpfd, 'w')
76 tmpfile = util.atomictempfile(storefilename,
77 createmode=self.repo.store.createmode)
85 78
86 79 try:
87 80 hhash = binascii.hexlify(self._getfile(tmpfile, filename, hash))
88 81 except StoreError, err:
89 82 ui.warn(err.longmessage())
90 83 hhash = ""
91 84
92 85 if hhash != hash:
93 86 if hhash != "":
94 87 ui.warn(_('%s: data corruption (expected %s, got %s)\n')
95 88 % (filename, hash, hhash))
96 tmpfile.close() # no-op if it's already closed
97 os.remove(tmpfilename)
89 tmpfile.discard() # no-op if it's already closed
98 90 missing.append(filename)
99 91 continue
100 92
101 if os.path.exists(storefilename): # Windows
102 os.remove(storefilename)
103 os.rename(tmpfilename, storefilename)
93 tmpfile.close()
104 94 lfutil.linktousercache(self.repo, hash)
105 95 success.append((filename, hhash))
106 96
107 97 ui.progress(_('getting largefiles'), None)
108 98 return (success, missing)
109 99
110 100 def verify(self, revs, contents=False):
111 101 '''Verify the existence (and, optionally, contents) of every big
112 102 file revision referenced by every changeset in revs.
113 103 Return 0 if all is well, non-zero on any errors.'''
114 104 write = self.ui.write
115 105 failed = False
116 106
117 107 write(_('searching %d changesets for largefiles\n') % len(revs))
118 108 verified = set() # set of (filename, filenode) tuples
119 109
120 110 for rev in revs:
121 111 cctx = self.repo[rev]
122 112 cset = "%d:%s" % (cctx.rev(), node.short(cctx.node()))
123 113
124 114 failed = util.any(self._verifyfile(
125 115 cctx, cset, contents, standin, verified) for standin in cctx)
126 116
127 117 num_revs = len(verified)
128 118 num_lfiles = len(set([fname for (fname, fnode) in verified]))
129 119 if contents:
130 120 write(_('verified contents of %d revisions of %d largefiles\n')
131 121 % (num_revs, num_lfiles))
132 122 else:
133 123 write(_('verified existence of %d revisions of %d largefiles\n')
134 124 % (num_revs, num_lfiles))
135 125
136 126 return int(failed)
137 127
138 128 def _getfile(self, tmpfile, filename, hash):
139 129 '''Fetch one revision of one file from the store and write it
140 130 to tmpfile. Compute the hash of the file on-the-fly as it
141 131 downloads and return the binary hash. Close tmpfile. Raise
142 132 StoreError if unable to download the file (e.g. it does not
143 133 exist in the store).'''
144 134 raise NotImplementedError('abstract method')
145 135
146 136 def _verifyfile(self, cctx, cset, contents, standin, verified):
147 137 '''Perform the actual verification of a file in the store.
148 138 '''
149 139 raise NotImplementedError('abstract method')
150 140
151 141 import localstore, wirestore
152 142
153 143 _storeprovider = {
154 144 'file': [localstore.localstore],
155 145 'http': [wirestore.wirestore],
156 146 'https': [wirestore.wirestore],
157 147 'ssh': [wirestore.wirestore],
158 148 }
159 149
160 150 _scheme_re = re.compile(r'^([a-zA-Z0-9+-.]+)://')
161 151
162 152 # During clone this function is passed the src's ui object
163 153 # but it needs the dest's ui object so it can read out of
164 154 # the config file. Use repo.ui instead.
165 155 def _openstore(repo, remote=None, put=False):
166 156 ui = repo.ui
167 157
168 158 if not remote:
169 159 lfpullsource = getattr(repo, 'lfpullsource', None)
170 160 if lfpullsource:
171 161 path = ui.expandpath(lfpullsource)
172 162 else:
173 163 path = ui.expandpath('default-push', 'default')
174 164
175 165 # ui.expandpath() leaves 'default-push' and 'default' alone if
176 166 # they cannot be expanded: fallback to the empty string,
177 167 # meaning the current directory.
178 168 if path == 'default-push' or path == 'default':
179 169 path = ''
180 170 remote = repo
181 171 else:
182 172 remote = hg.peer(repo, {}, path)
183 173
184 174 # The path could be a scheme so use Mercurial's normal functionality
185 175 # to resolve the scheme to a repository and use its path
186 176 path = util.safehasattr(remote, 'url') and remote.url() or remote.path
187 177
188 178 match = _scheme_re.match(path)
189 179 if not match: # regular filesystem path
190 180 scheme = 'file'
191 181 else:
192 182 scheme = match.group(1)
193 183
194 184 try:
195 185 storeproviders = _storeprovider[scheme]
196 186 except KeyError:
197 187 raise util.Abort(_('unsupported URL scheme %r') % scheme)
198 188
199 189 for class_obj in storeproviders:
200 190 try:
201 191 return class_obj(ui, repo, remote)
202 192 except lfutil.storeprotonotcapable:
203 193 pass
204 194
205 195 raise util.Abort(_('%s does not appear to be a largefile store') % path)
@@ -1,96 +1,105 b''
1 1 $ "$TESTDIR/hghave" unix-permissions || exit 80
2 2
3 3 Create user cache directory
4 4
5 5 $ USERCACHE=`pwd`/cache; export USERCACHE
6 6 $ cat <<EOF >> ${HGRCPATH}
7 7 > [extensions]
8 8 > hgext.largefiles=
9 9 > [largefiles]
10 10 > usercache=${USERCACHE}
11 11 > EOF
12 12 $ mkdir -p ${USERCACHE}
13 13
14 14 Create source repo, and commit adding largefile.
15 15
16 16 $ hg init src
17 17 $ cd src
18 18 $ echo large > large
19 19 $ hg add --large large
20 20 $ hg commit -m 'add largefile'
21 21 $ cd ..
22 22
23 23 Discard all cached largefiles in USERCACHE
24 24
25 25 $ rm -rf ${USERCACHE}
26 26
27 27 Create mirror repo, and pull from source without largefile:
28 28 "pull" is used instead of "clone" for suppression of (1) updating to
29 29 tip (= cahcing largefile from source repo), and (2) recording source
30 30 repo as "default" path in .hg/hgrc.
31 31
32 32 $ hg init mirror
33 33 $ cd mirror
34 34 $ hg pull ../src
35 35 pulling from ../src
36 36 requesting all changes
37 37 adding changesets
38 38 adding manifests
39 39 adding file changes
40 40 added 1 changesets with 1 changes to 1 files
41 41 (run 'hg update' to get a working copy)
42 42 caching new largefiles
43 43 0 largefiles cached
44 44
45 45 Update working directory to "tip", which requires largefile("large"),
46 46 but there is no cache file for it. So, hg must treat it as
47 47 "missing"(!) file.
48 48
49 49 $ hg update
50 50 1 files updated, 0 files merged, 0 files removed, 0 files unresolved
51 51 getting changed largefiles
52 52 large: Can't get file locally
53 53 (no default or default-push path set in hgrc)
54 54 0 largefiles updated, 0 removed
55 55 $ hg status
56 56 ! large
57 57
58 58 Update working directory to null: this cleanup .hg/largefiles/dirstate
59 59
60 60 $ hg update null
61 61 0 files updated, 0 files merged, 1 files removed, 0 files unresolved
62 62 getting changed largefiles
63 63 0 largefiles updated, 0 removed
64 64
65 65 Update working directory to tip, again.
66 66
67 67 $ hg update
68 68 1 files updated, 0 files merged, 0 files removed, 0 files unresolved
69 69 getting changed largefiles
70 70 large: Can't get file locally
71 71 (no default or default-push path set in hgrc)
72 72 0 largefiles updated, 0 removed
73 73 $ hg status
74 74 ! large
75 75
76 76 Portable way to print file permissions:
77 77
78 78 $ cd ..
79 79 $ cat > ls-l.py <<EOF
80 80 > #!/usr/bin/env python
81 81 > import sys, os
82 82 > path = sys.argv[1]
83 83 > print '%03o' % (os.lstat(path).st_mode & 0777)
84 84 > EOF
85 85 $ chmod +x ls-l.py
86 86
87 87 Test that files in .hg/largefiles inherit mode from .hg/store, not
88 88 from file in working copy:
89 89
90 90 $ cd src
91 91 $ chmod 750 .hg/store
92 92 $ chmod 660 large
93 93 $ echo change >> large
94 94 $ hg commit -m change
95 95 $ ../ls-l.py .hg/largefiles/e151b474069de4ca6898f67ce2f2a7263adf8fea
96 96 640
97
98 Test permission of with files in .hg/largefiles created by update:
99
100 $ cd ../mirror
101 $ rm -r "$USERCACHE" .hg/largefiles # avoid links
102 $ chmod 750 .hg/store
103 $ hg pull ../src --update -q
104 $ ../ls-l.py .hg/largefiles/e151b474069de4ca6898f67ce2f2a7263adf8fea
105 640
General Comments 0
You need to be logged in to leave comments. Login now