##// END OF EJS Templates
largefiles: use strip() instead of slicing to get rid of EOL of standin...
FUJIWARA Katsunori -
r31658:f1cf6a74 default
parent child Browse files
Show More
@@ -1,164 +1,164
1 1 # Copyright 2009-2010 Gregory P. Ward
2 2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
3 3 # Copyright 2010-2011 Fog Creek Software
4 4 # Copyright 2010-2011 Unity Technologies
5 5 #
6 6 # This software may be used and distributed according to the terms of the
7 7 # GNU General Public License version 2 or any later version.
8 8
9 9 '''base class for store implementations and store-related utility code'''
10 10 from __future__ import absolute_import
11 11
12 12 from mercurial.i18n import _
13 13
14 14 from mercurial import node, util
15 15
16 16 from . import lfutil
17 17
18 18 class StoreError(Exception):
19 19 '''Raised when there is a problem getting files from or putting
20 20 files to a central store.'''
21 21 def __init__(self, filename, hash, url, detail):
22 22 self.filename = filename
23 23 self.hash = hash
24 24 self.url = url
25 25 self.detail = detail
26 26
27 27 def longmessage(self):
28 28 return (_("error getting id %s from url %s for file %s: %s\n") %
29 29 (self.hash, util.hidepassword(self.url), self.filename,
30 30 self.detail))
31 31
32 32 def __str__(self):
33 33 return "%s: %s" % (util.hidepassword(self.url), self.detail)
34 34
35 35 class basestore(object):
36 36 def __init__(self, ui, repo, url):
37 37 self.ui = ui
38 38 self.repo = repo
39 39 self.url = url
40 40
41 41 def put(self, source, hash):
42 42 '''Put source file into the store so it can be retrieved by hash.'''
43 43 raise NotImplementedError('abstract method')
44 44
45 45 def exists(self, hashes):
46 46 '''Check to see if the store contains the given hashes. Given an
47 47 iterable of hashes it returns a mapping from hash to bool.'''
48 48 raise NotImplementedError('abstract method')
49 49
50 50 def get(self, files):
51 51 '''Get the specified largefiles from the store and write to local
52 52 files under repo.root. files is a list of (filename, hash)
53 53 tuples. Return (success, missing), lists of files successfully
54 54 downloaded and those not found in the store. success is a list
55 55 of (filename, hash) tuples; missing is a list of filenames that
56 56 we could not get. (The detailed error message will already have
57 57 been presented to the user, so missing is just supplied as a
58 58 summary.)'''
59 59 success = []
60 60 missing = []
61 61 ui = self.ui
62 62
63 63 at = 0
64 64 available = self.exists(set(hash for (_filename, hash) in files))
65 65 for filename, hash in files:
66 66 ui.progress(_('getting largefiles'), at, unit=_('files'),
67 67 total=len(files))
68 68 at += 1
69 69 ui.note(_('getting %s:%s\n') % (filename, hash))
70 70
71 71 if not available.get(hash):
72 72 ui.warn(_('%s: largefile %s not available from %s\n')
73 73 % (filename, hash, util.hidepassword(self.url)))
74 74 missing.append(filename)
75 75 continue
76 76
77 77 if self._gethash(filename, hash):
78 78 success.append((filename, hash))
79 79 else:
80 80 missing.append(filename)
81 81
82 82 ui.progress(_('getting largefiles'), None)
83 83 return (success, missing)
84 84
85 85 def _gethash(self, filename, hash):
86 86 """Get file with the provided hash and store it in the local repo's
87 87 store and in the usercache.
88 88 filename is for informational messages only.
89 89 """
90 90 util.makedirs(lfutil.storepath(self.repo, ''))
91 91 storefilename = lfutil.storepath(self.repo, hash)
92 92
93 93 tmpname = storefilename + '.tmp'
94 94 with util.atomictempfile(tmpname,
95 95 createmode=self.repo.store.createmode) as tmpfile:
96 96 try:
97 97 gothash = self._getfile(tmpfile, filename, hash)
98 98 except StoreError as err:
99 99 self.ui.warn(err.longmessage())
100 100 gothash = ""
101 101
102 102 if gothash != hash:
103 103 if gothash != "":
104 104 self.ui.warn(_('%s: data corruption (expected %s, got %s)\n')
105 105 % (filename, hash, gothash))
106 106 util.unlink(tmpname)
107 107 return False
108 108
109 109 util.rename(tmpname, storefilename)
110 110 lfutil.linktousercache(self.repo, hash)
111 111 return True
112 112
113 113 def verify(self, revs, contents=False):
114 114 '''Verify the existence (and, optionally, contents) of every big
115 115 file revision referenced by every changeset in revs.
116 116 Return 0 if all is well, non-zero on any errors.'''
117 117
118 118 self.ui.status(_('searching %d changesets for largefiles\n') %
119 119 len(revs))
120 120 verified = set() # set of (filename, filenode) tuples
121 121 filestocheck = [] # list of (cset, filename, expectedhash)
122 122 for rev in revs:
123 123 cctx = self.repo[rev]
124 124 cset = "%d:%s" % (cctx.rev(), node.short(cctx.node()))
125 125
126 126 for standin in cctx:
127 127 filename = lfutil.splitstandin(standin)
128 128 if filename:
129 129 fctx = cctx[standin]
130 130 key = (filename, fctx.filenode())
131 131 if key not in verified:
132 132 verified.add(key)
133 expectedhash = fctx.data()[0:40]
133 expectedhash = fctx.data().strip()
134 134 filestocheck.append((cset, filename, expectedhash))
135 135
136 136 failed = self._verifyfiles(contents, filestocheck)
137 137
138 138 numrevs = len(verified)
139 139 numlfiles = len(set([fname for (fname, fnode) in verified]))
140 140 if contents:
141 141 self.ui.status(
142 142 _('verified contents of %d revisions of %d largefiles\n')
143 143 % (numrevs, numlfiles))
144 144 else:
145 145 self.ui.status(
146 146 _('verified existence of %d revisions of %d largefiles\n')
147 147 % (numrevs, numlfiles))
148 148 return int(failed)
149 149
150 150 def _getfile(self, tmpfile, filename, hash):
151 151 '''Fetch one revision of one file from the store and write it
152 152 to tmpfile. Compute the hash of the file on-the-fly as it
153 153 downloads and return the hash. Close tmpfile. Raise
154 154 StoreError if unable to download the file (e.g. it does not
155 155 exist in the store).'''
156 156 raise NotImplementedError('abstract method')
157 157
158 158 def _verifyfiles(self, contents, filestocheck):
159 159 '''Perform the actual verification of files in the store.
160 160 'contents' controls verification of content hash.
161 161 'filestocheck' is list of files to check.
162 162 Returns _true_ if any problems are found!
163 163 '''
164 164 raise NotImplementedError('abstract method')
General Comments 0
You need to be logged in to leave comments. Login now