##// END OF EJS Templates
largefiles: use strip() instead of slicing to get rid of EOL of standin...
FUJIWARA Katsunori -
r31658:f1cf6a74 default
parent child Browse files
Show More
@@ -1,164 +1,164
1 # Copyright 2009-2010 Gregory P. Ward
1 # Copyright 2009-2010 Gregory P. Ward
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
3 # Copyright 2010-2011 Fog Creek Software
3 # Copyright 2010-2011 Fog Creek Software
4 # Copyright 2010-2011 Unity Technologies
4 # Copyright 2010-2011 Unity Technologies
5 #
5 #
6 # This software may be used and distributed according to the terms of the
6 # This software may be used and distributed according to the terms of the
7 # GNU General Public License version 2 or any later version.
7 # GNU General Public License version 2 or any later version.
8
8
9 '''base class for store implementations and store-related utility code'''
9 '''base class for store implementations and store-related utility code'''
10 from __future__ import absolute_import
10 from __future__ import absolute_import
11
11
12 from mercurial.i18n import _
12 from mercurial.i18n import _
13
13
14 from mercurial import node, util
14 from mercurial import node, util
15
15
16 from . import lfutil
16 from . import lfutil
17
17
18 class StoreError(Exception):
18 class StoreError(Exception):
19 '''Raised when there is a problem getting files from or putting
19 '''Raised when there is a problem getting files from or putting
20 files to a central store.'''
20 files to a central store.'''
21 def __init__(self, filename, hash, url, detail):
21 def __init__(self, filename, hash, url, detail):
22 self.filename = filename
22 self.filename = filename
23 self.hash = hash
23 self.hash = hash
24 self.url = url
24 self.url = url
25 self.detail = detail
25 self.detail = detail
26
26
27 def longmessage(self):
27 def longmessage(self):
28 return (_("error getting id %s from url %s for file %s: %s\n") %
28 return (_("error getting id %s from url %s for file %s: %s\n") %
29 (self.hash, util.hidepassword(self.url), self.filename,
29 (self.hash, util.hidepassword(self.url), self.filename,
30 self.detail))
30 self.detail))
31
31
32 def __str__(self):
32 def __str__(self):
33 return "%s: %s" % (util.hidepassword(self.url), self.detail)
33 return "%s: %s" % (util.hidepassword(self.url), self.detail)
34
34
35 class basestore(object):
35 class basestore(object):
36 def __init__(self, ui, repo, url):
36 def __init__(self, ui, repo, url):
37 self.ui = ui
37 self.ui = ui
38 self.repo = repo
38 self.repo = repo
39 self.url = url
39 self.url = url
40
40
41 def put(self, source, hash):
41 def put(self, source, hash):
42 '''Put source file into the store so it can be retrieved by hash.'''
42 '''Put source file into the store so it can be retrieved by hash.'''
43 raise NotImplementedError('abstract method')
43 raise NotImplementedError('abstract method')
44
44
45 def exists(self, hashes):
45 def exists(self, hashes):
46 '''Check to see if the store contains the given hashes. Given an
46 '''Check to see if the store contains the given hashes. Given an
47 iterable of hashes it returns a mapping from hash to bool.'''
47 iterable of hashes it returns a mapping from hash to bool.'''
48 raise NotImplementedError('abstract method')
48 raise NotImplementedError('abstract method')
49
49
50 def get(self, files):
50 def get(self, files):
51 '''Get the specified largefiles from the store and write to local
51 '''Get the specified largefiles from the store and write to local
52 files under repo.root. files is a list of (filename, hash)
52 files under repo.root. files is a list of (filename, hash)
53 tuples. Return (success, missing), lists of files successfully
53 tuples. Return (success, missing), lists of files successfully
54 downloaded and those not found in the store. success is a list
54 downloaded and those not found in the store. success is a list
55 of (filename, hash) tuples; missing is a list of filenames that
55 of (filename, hash) tuples; missing is a list of filenames that
56 we could not get. (The detailed error message will already have
56 we could not get. (The detailed error message will already have
57 been presented to the user, so missing is just supplied as a
57 been presented to the user, so missing is just supplied as a
58 summary.)'''
58 summary.)'''
59 success = []
59 success = []
60 missing = []
60 missing = []
61 ui = self.ui
61 ui = self.ui
62
62
63 at = 0
63 at = 0
64 available = self.exists(set(hash for (_filename, hash) in files))
64 available = self.exists(set(hash for (_filename, hash) in files))
65 for filename, hash in files:
65 for filename, hash in files:
66 ui.progress(_('getting largefiles'), at, unit=_('files'),
66 ui.progress(_('getting largefiles'), at, unit=_('files'),
67 total=len(files))
67 total=len(files))
68 at += 1
68 at += 1
69 ui.note(_('getting %s:%s\n') % (filename, hash))
69 ui.note(_('getting %s:%s\n') % (filename, hash))
70
70
71 if not available.get(hash):
71 if not available.get(hash):
72 ui.warn(_('%s: largefile %s not available from %s\n')
72 ui.warn(_('%s: largefile %s not available from %s\n')
73 % (filename, hash, util.hidepassword(self.url)))
73 % (filename, hash, util.hidepassword(self.url)))
74 missing.append(filename)
74 missing.append(filename)
75 continue
75 continue
76
76
77 if self._gethash(filename, hash):
77 if self._gethash(filename, hash):
78 success.append((filename, hash))
78 success.append((filename, hash))
79 else:
79 else:
80 missing.append(filename)
80 missing.append(filename)
81
81
82 ui.progress(_('getting largefiles'), None)
82 ui.progress(_('getting largefiles'), None)
83 return (success, missing)
83 return (success, missing)
84
84
85 def _gethash(self, filename, hash):
85 def _gethash(self, filename, hash):
86 """Get file with the provided hash and store it in the local repo's
86 """Get file with the provided hash and store it in the local repo's
87 store and in the usercache.
87 store and in the usercache.
88 filename is for informational messages only.
88 filename is for informational messages only.
89 """
89 """
90 util.makedirs(lfutil.storepath(self.repo, ''))
90 util.makedirs(lfutil.storepath(self.repo, ''))
91 storefilename = lfutil.storepath(self.repo, hash)
91 storefilename = lfutil.storepath(self.repo, hash)
92
92
93 tmpname = storefilename + '.tmp'
93 tmpname = storefilename + '.tmp'
94 with util.atomictempfile(tmpname,
94 with util.atomictempfile(tmpname,
95 createmode=self.repo.store.createmode) as tmpfile:
95 createmode=self.repo.store.createmode) as tmpfile:
96 try:
96 try:
97 gothash = self._getfile(tmpfile, filename, hash)
97 gothash = self._getfile(tmpfile, filename, hash)
98 except StoreError as err:
98 except StoreError as err:
99 self.ui.warn(err.longmessage())
99 self.ui.warn(err.longmessage())
100 gothash = ""
100 gothash = ""
101
101
102 if gothash != hash:
102 if gothash != hash:
103 if gothash != "":
103 if gothash != "":
104 self.ui.warn(_('%s: data corruption (expected %s, got %s)\n')
104 self.ui.warn(_('%s: data corruption (expected %s, got %s)\n')
105 % (filename, hash, gothash))
105 % (filename, hash, gothash))
106 util.unlink(tmpname)
106 util.unlink(tmpname)
107 return False
107 return False
108
108
109 util.rename(tmpname, storefilename)
109 util.rename(tmpname, storefilename)
110 lfutil.linktousercache(self.repo, hash)
110 lfutil.linktousercache(self.repo, hash)
111 return True
111 return True
112
112
113 def verify(self, revs, contents=False):
113 def verify(self, revs, contents=False):
114 '''Verify the existence (and, optionally, contents) of every big
114 '''Verify the existence (and, optionally, contents) of every big
115 file revision referenced by every changeset in revs.
115 file revision referenced by every changeset in revs.
116 Return 0 if all is well, non-zero on any errors.'''
116 Return 0 if all is well, non-zero on any errors.'''
117
117
118 self.ui.status(_('searching %d changesets for largefiles\n') %
118 self.ui.status(_('searching %d changesets for largefiles\n') %
119 len(revs))
119 len(revs))
120 verified = set() # set of (filename, filenode) tuples
120 verified = set() # set of (filename, filenode) tuples
121 filestocheck = [] # list of (cset, filename, expectedhash)
121 filestocheck = [] # list of (cset, filename, expectedhash)
122 for rev in revs:
122 for rev in revs:
123 cctx = self.repo[rev]
123 cctx = self.repo[rev]
124 cset = "%d:%s" % (cctx.rev(), node.short(cctx.node()))
124 cset = "%d:%s" % (cctx.rev(), node.short(cctx.node()))
125
125
126 for standin in cctx:
126 for standin in cctx:
127 filename = lfutil.splitstandin(standin)
127 filename = lfutil.splitstandin(standin)
128 if filename:
128 if filename:
129 fctx = cctx[standin]
129 fctx = cctx[standin]
130 key = (filename, fctx.filenode())
130 key = (filename, fctx.filenode())
131 if key not in verified:
131 if key not in verified:
132 verified.add(key)
132 verified.add(key)
133 expectedhash = fctx.data()[0:40]
133 expectedhash = fctx.data().strip()
134 filestocheck.append((cset, filename, expectedhash))
134 filestocheck.append((cset, filename, expectedhash))
135
135
136 failed = self._verifyfiles(contents, filestocheck)
136 failed = self._verifyfiles(contents, filestocheck)
137
137
138 numrevs = len(verified)
138 numrevs = len(verified)
139 numlfiles = len(set([fname for (fname, fnode) in verified]))
139 numlfiles = len(set([fname for (fname, fnode) in verified]))
140 if contents:
140 if contents:
141 self.ui.status(
141 self.ui.status(
142 _('verified contents of %d revisions of %d largefiles\n')
142 _('verified contents of %d revisions of %d largefiles\n')
143 % (numrevs, numlfiles))
143 % (numrevs, numlfiles))
144 else:
144 else:
145 self.ui.status(
145 self.ui.status(
146 _('verified existence of %d revisions of %d largefiles\n')
146 _('verified existence of %d revisions of %d largefiles\n')
147 % (numrevs, numlfiles))
147 % (numrevs, numlfiles))
148 return int(failed)
148 return int(failed)
149
149
150 def _getfile(self, tmpfile, filename, hash):
150 def _getfile(self, tmpfile, filename, hash):
151 '''Fetch one revision of one file from the store and write it
151 '''Fetch one revision of one file from the store and write it
152 to tmpfile. Compute the hash of the file on-the-fly as it
152 to tmpfile. Compute the hash of the file on-the-fly as it
153 downloads and return the hash. Close tmpfile. Raise
153 downloads and return the hash. Close tmpfile. Raise
154 StoreError if unable to download the file (e.g. it does not
154 StoreError if unable to download the file (e.g. it does not
155 exist in the store).'''
155 exist in the store).'''
156 raise NotImplementedError('abstract method')
156 raise NotImplementedError('abstract method')
157
157
158 def _verifyfiles(self, contents, filestocheck):
158 def _verifyfiles(self, contents, filestocheck):
159 '''Perform the actual verification of files in the store.
159 '''Perform the actual verification of files in the store.
160 'contents' controls verification of content hash.
160 'contents' controls verification of content hash.
161 'filestocheck' is list of files to check.
161 'filestocheck' is list of files to check.
162 Returns _true_ if any problems are found!
162 Returns _true_ if any problems are found!
163 '''
163 '''
164 raise NotImplementedError('abstract method')
164 raise NotImplementedError('abstract method')
General Comments 0
You need to be logged in to leave comments. Login now