Show More
The requested changes are too big and content was truncated. Show full diff
@@ -1,563 +1,563 b'' | |||
|
1 | 1 | #!/usr/bin/env python |
|
2 | 2 | # |
|
3 | 3 | # check-code - a style and portability checker for Mercurial |
|
4 | 4 | # |
|
5 | 5 | # Copyright 2010 Matt Mackall <mpm@selenic.com> |
|
6 | 6 | # |
|
7 | 7 | # This software may be used and distributed according to the terms of the |
|
8 | 8 | # GNU General Public License version 2 or any later version. |
|
9 | 9 | |
|
10 | 10 | """style and portability checker for Mercurial |
|
11 | 11 | |
|
12 | 12 | when a rule triggers wrong, do one of the following (prefer one from top): |
|
13 | 13 | * do the work-around the rule suggests |
|
14 | 14 | * doublecheck that it is a false match |
|
15 | 15 | * improve the rule pattern |
|
16 | 16 | * add an ignore pattern to the rule (3rd arg) which matches your good line |
|
17 | 17 | (you can append a short comment and match this, like: #re-raises, # no-py24) |
|
18 | 18 | * change the pattern to a warning and list the exception in test-check-code-hg |
|
19 | 19 | * ONLY use no--check-code for skipping entire files from external sources |
|
20 | 20 | """ |
|
21 | 21 | |
|
22 | 22 | import re, glob, os, sys |
|
23 | 23 | import keyword |
|
24 | 24 | import optparse |
|
25 | 25 | try: |
|
26 | 26 | import re2 |
|
27 | 27 | except ImportError: |
|
28 | 28 | re2 = None |
|
29 | 29 | |
|
30 | 30 | def compilere(pat, multiline=False): |
|
31 | 31 | if multiline: |
|
32 | 32 | pat = '(?m)' + pat |
|
33 | 33 | if re2: |
|
34 | 34 | try: |
|
35 | 35 | return re2.compile(pat) |
|
36 | 36 | except re2.error: |
|
37 | 37 | pass |
|
38 | 38 | return re.compile(pat) |
|
39 | 39 | |
|
40 | 40 | def repquote(m): |
|
41 | 41 | fromc = '.:' |
|
42 | 42 | tochr = 'pq' |
|
43 | 43 | def encodechr(i): |
|
44 | 44 | if i > 255: |
|
45 | 45 | return 'u' |
|
46 | 46 | c = chr(i) |
|
47 | 47 | if c in ' \n': |
|
48 | 48 | return c |
|
49 | 49 | if c.isalpha(): |
|
50 | 50 | return 'x' |
|
51 | 51 | if c.isdigit(): |
|
52 | 52 | return 'n' |
|
53 | 53 | try: |
|
54 | 54 | return tochr[fromc.find(c)] |
|
55 | 55 | except (ValueError, IndexError): |
|
56 | 56 | return 'o' |
|
57 | 57 | t = m.group('text') |
|
58 | 58 | tt = ''.join(encodechr(i) for i in xrange(256)) |
|
59 | 59 | t = t.translate(tt) |
|
60 | 60 | return m.group('quote') + t + m.group('quote') |
|
61 | 61 | |
|
62 | 62 | def reppython(m): |
|
63 | 63 | comment = m.group('comment') |
|
64 | 64 | if comment: |
|
65 | 65 | l = len(comment.rstrip()) |
|
66 | 66 | return "#" * l + comment[l:] |
|
67 | 67 | return repquote(m) |
|
68 | 68 | |
|
69 | 69 | def repcomment(m): |
|
70 | 70 | return m.group(1) + "#" * len(m.group(2)) |
|
71 | 71 | |
|
72 | 72 | def repccomment(m): |
|
73 | 73 | t = re.sub(r"((?<=\n) )|\S", "x", m.group(2)) |
|
74 | 74 | return m.group(1) + t + "*/" |
|
75 | 75 | |
|
76 | 76 | def repcallspaces(m): |
|
77 | 77 | t = re.sub(r"\n\s+", "\n", m.group(2)) |
|
78 | 78 | return m.group(1) + t |
|
79 | 79 | |
|
80 | 80 | def repinclude(m): |
|
81 | 81 | return m.group(1) + "<foo>" |
|
82 | 82 | |
|
83 | 83 | def rephere(m): |
|
84 | 84 | t = re.sub(r"\S", "x", m.group(2)) |
|
85 | 85 | return m.group(1) + t |
|
86 | 86 | |
|
87 | 87 | |
|
88 | 88 | testpats = [ |
|
89 | 89 | [ |
|
90 | 90 | (r'pushd|popd', "don't use 'pushd' or 'popd', use 'cd'"), |
|
91 | 91 | (r'\W\$?\(\([^\)\n]*\)\)', "don't use (()) or $(()), use 'expr'"), |
|
92 | 92 | (r'grep.*-q', "don't use 'grep -q', redirect to /dev/null"), |
|
93 | 93 | (r'(?<!hg )grep.*-a', "don't use 'grep -a', use in-line python"), |
|
94 | 94 | (r'sed.*-i', "don't use 'sed -i', use a temporary file"), |
|
95 | 95 | (r'\becho\b.*\\n', "don't use 'echo \\n', use printf"), |
|
96 | 96 | (r'echo -n', "don't use 'echo -n', use printf"), |
|
97 | 97 | (r'(^| )wc[^|]*$\n(?!.*\(re\))', "filter wc output"), |
|
98 | 98 | (r'head -c', "don't use 'head -c', use 'dd'"), |
|
99 | 99 | (r'tail -n', "don't use the '-n' option to tail, just use '-<num>'"), |
|
100 | 100 | (r'sha1sum', "don't use sha1sum, use $TESTDIR/md5sum.py"), |
|
101 | 101 | (r'ls.*-\w*R', "don't use 'ls -R', use 'find'"), |
|
102 | 102 | (r'printf.*[^\\]\\([1-9]|0\d)', "don't use 'printf \NNN', use Python"), |
|
103 | 103 | (r'printf.*[^\\]\\x', "don't use printf \\x, use Python"), |
|
104 | 104 | (r'\$\(.*\)', "don't use $(expr), use `expr`"), |
|
105 | 105 | (r'rm -rf \*', "don't use naked rm -rf, target a directory"), |
|
106 | 106 | (r'(^|\|\s*)grep (-\w\s+)*[^|]*[(|]\w', |
|
107 | 107 | "use egrep for extended grep syntax"), |
|
108 | 108 | (r'/bin/', "don't use explicit paths for tools"), |
|
109 | 109 | (r'[^\n]\Z', "no trailing newline"), |
|
110 | 110 | (r'export.*=', "don't export and assign at once"), |
|
111 | 111 | (r'^source\b', "don't use 'source', use '.'"), |
|
112 | 112 | (r'touch -d', "don't use 'touch -d', use 'touch -t' instead"), |
|
113 | 113 | (r'ls +[^|\n-]+ +-', "options to 'ls' must come before filenames"), |
|
114 | 114 | (r'[^>\n]>\s*\$HGRCPATH', "don't overwrite $HGRCPATH, append to it"), |
|
115 | 115 | (r'^stop\(\)', "don't use 'stop' as a shell function name"), |
|
116 | 116 | (r'(\[|\btest\b).*-e ', "don't use 'test -e', use 'test -f'"), |
|
117 | 117 | (r'^alias\b.*=', "don't use alias, use a function"), |
|
118 | 118 | (r'if\s*!', "don't use '!' to negate exit status"), |
|
119 | 119 | (r'/dev/u?random', "don't use entropy, use /dev/zero"), |
|
120 | 120 | (r'do\s*true;\s*done', "don't use true as loop body, use sleep 0"), |
|
121 | 121 | (r'^( *)\t', "don't use tabs to indent"), |
|
122 | 122 | (r'sed (-e )?\'(\d+|/[^/]*/)i(?!\\\n)', |
|
123 | 123 | "put a backslash-escaped newline after sed 'i' command"), |
|
124 | 124 | (r'^diff *-\w*u.*$\n(^ \$ |^$)', "prefix diff -u with cmp"), |
|
125 | 125 | ], |
|
126 | 126 | # warnings |
|
127 | 127 | [ |
|
128 | 128 | (r'^function', "don't use 'function', use old style"), |
|
129 | 129 | (r'^diff.*-\w*N', "don't use 'diff -N'"), |
|
130 | 130 | (r'\$PWD|\${PWD}', "don't use $PWD, use `pwd`"), |
|
131 | 131 | (r'^([^"\'\n]|("[^"\n]*")|(\'[^\'\n]*\'))*\^', "^ must be quoted"), |
|
132 | 132 | (r'kill (`|\$\()', "don't use kill, use killdaemons.py") |
|
133 | 133 | ] |
|
134 | 134 | ] |
|
135 | 135 | |
|
136 | 136 | testfilters = [ |
|
137 | 137 | (r"( *)(#([^\n]*\S)?)", repcomment), |
|
138 | 138 | (r"<<(\S+)((.|\n)*?\n\1)", rephere), |
|
139 | 139 | ] |
|
140 | 140 | |
|
141 | 141 | winglobmsg = "use (glob) to match Windows paths too" |
|
142 | 142 | uprefix = r"^ \$ " |
|
143 | 143 | utestpats = [ |
|
144 | 144 | [ |
|
145 | 145 | (r'^(\S.*|| [$>] .*)[ \t]\n', "trailing whitespace on non-output"), |
|
146 | 146 | (uprefix + r'.*\|\s*sed[^|>\n]*\n', |
|
147 | 147 | "use regex test output patterns instead of sed"), |
|
148 | 148 | (uprefix + r'(true|exit 0)', "explicit zero exit unnecessary"), |
|
149 | 149 | (uprefix + r'.*(?<!\[)\$\?', "explicit exit code checks unnecessary"), |
|
150 | 150 | (uprefix + r'.*\|\| echo.*(fail|error)', |
|
151 | 151 | "explicit exit code checks unnecessary"), |
|
152 | 152 | (uprefix + r'set -e', "don't use set -e"), |
|
153 | 153 | (uprefix + r'(\s|fi\b|done\b)', "use > for continued lines"), |
|
154 | 154 | (uprefix + r'.*:\.\S*/', "x:.y in a path does not work on msys, rewrite " |
|
155 | 155 | "as x://.y, or see `hg log -k msys` for alternatives", r'-\S+:\.|' #-Rxxx |
|
156 | 156 | 'hg pull -q file:../test'), # in test-pull.t which is skipped on windows |
|
157 | 157 | (r'^ saved backup bundle to \$TESTTMP.*\.hg$', winglobmsg), |
|
158 | 158 | (r'^ changeset .* references (corrupted|missing) \$TESTTMP/.*[^)]$', |
|
159 | 159 | winglobmsg), |
|
160 | 160 | (r'^ pulling from \$TESTTMP/.*[^)]$', winglobmsg, |
|
161 | 161 | '\$TESTTMP/unix-repo$'), # in test-issue1802.t which skipped on windows |
|
162 | 162 | (r'^ reverting .*/.*[^)]$', winglobmsg), |
|
163 | 163 | (r'^ cloning subrepo \S+/.*[^)]$', winglobmsg), |
|
164 | 164 | (r'^ pushing to \$TESTTMP/.*[^)]$', winglobmsg), |
|
165 | 165 | (r'^ pushing subrepo \S+/\S+ to.*[^)]$', winglobmsg), |
|
166 | 166 | (r'^ moving \S+/.*[^)]$', winglobmsg), |
|
167 | 167 | (r'^ no changes made to subrepo since.*/.*[^)]$', winglobmsg), |
|
168 | 168 | (r'^ .*: largefile \S+ not available from file:.*/.*[^)]$', winglobmsg), |
|
169 | 169 | (r'^ .*file://\$TESTTMP', |
|
170 | 170 | 'write "file:/*/$TESTTMP" + (glob) to match on windows too'), |
|
171 | 171 | ], |
|
172 | 172 | # warnings |
|
173 | 173 | [ |
|
174 | 174 | (r'^ [^*?/\n]* \(glob\)$', |
|
175 | 175 | "glob match with no glob character (?*/)"), |
|
176 | 176 | ] |
|
177 | 177 | ] |
|
178 | 178 | |
|
179 | 179 | for i in [0, 1]: |
|
180 | 180 | for p, m in testpats[i]: |
|
181 | 181 | if p.startswith(r'^'): |
|
182 | 182 | p = r"^ [$>] (%s)" % p[1:] |
|
183 | 183 | else: |
|
184 | 184 | p = r"^ [$>] .*(%s)" % p |
|
185 | 185 | utestpats[i].append((p, m)) |
|
186 | 186 | |
|
187 | 187 | utestfilters = [ |
|
188 | 188 | (r"<<(\S+)((.|\n)*?\n > \1)", rephere), |
|
189 | 189 | (r"( *)(#([^\n]*\S)?)", repcomment), |
|
190 | 190 | ] |
|
191 | 191 | |
|
192 | 192 | pypats = [ |
|
193 | 193 | [ |
|
194 | 194 | (r'\([^)]*\*\w[^()]+\w+=', "can't pass varargs with keyword in Py2.5"), |
|
195 | 195 | (r'^\s*def\s*\w+\s*\(.*,\s*\(', |
|
196 | 196 | "tuple parameter unpacking not available in Python 3+"), |
|
197 | 197 | (r'lambda\s*\(.*,.*\)', |
|
198 | 198 | "tuple parameter unpacking not available in Python 3+"), |
|
199 | 199 | (r'import (.+,[^.]+\.[^.]+|[^.]+\.[^.]+,)', |
|
200 | 200 | '2to3 can\'t always rewrite "import qux, foo.bar", ' |
|
201 | 201 | 'use "import foo.bar" on its own line instead.'), |
|
202 | 202 | (r'(?<!def)\s+(cmp)\(', "cmp is not available in Python 3+"), |
|
203 | 203 | (r'\breduce\s*\(.*', "reduce is not available in Python 3+"), |
|
204 | 204 | (r'dict\(.*=', 'dict() is different in Py2 and 3 and is slower than {}', |
|
205 | 205 | 'dict-from-generator'), |
|
206 | 206 | (r'\.has_key\b', "dict.has_key is not available in Python 3+"), |
|
207 | 207 | (r'\s<>\s', '<> operator is not available in Python 3+, use !='), |
|
208 | 208 | (r'^\s*\t', "don't use tabs"), |
|
209 | 209 | (r'\S;\s*\n', "semicolon"), |
|
210 |
(r'[^_]_\("[^"]+"\ |
|
|
211 |
(r"[^_]_\('[^']+'\ |
|
|
210 | (r'[^_]_\((?:"[^"]+"[ \t\n+]*)+%', "don't use % inside _()"), | |
|
211 | (r"[^_]_\((?:'[^']+'[ \t\n+]*)+%", "don't use % inside _()"), | |
|
212 | 212 | (r'(\w|\)),\w', "missing whitespace after ,"), |
|
213 | 213 | (r'(\w|\))[+/*\-<>]\w', "missing whitespace in expression"), |
|
214 | 214 | (r'^\s+(\w|\.)+=\w[^,()\n]*$', "missing whitespace in assignment"), |
|
215 | 215 | (r'(\s+)try:\n((?:\n|\1\s.*\n)+?)\1except.*?:\n' |
|
216 | 216 | r'((?:\n|\1\s.*\n)+?)\1finally:', 'no try/except/finally in Python 2.4'), |
|
217 | 217 | (r'(?<!def)(\s+|^|\()next\(.+\)', |
|
218 | 218 | 'no next(foo) in Python 2.4 and 2.5, use foo.next() instead'), |
|
219 | 219 | (r'(\s+)try:\n((?:\n|\1\s.*\n)*?)\1\s*yield\b.*?' |
|
220 | 220 | r'((?:\n|\1\s.*\n)+?)\1finally:', |
|
221 | 221 | 'no yield inside try/finally in Python 2.4'), |
|
222 | 222 | (r'.{81}', "line too long"), |
|
223 | 223 | (r' x+[xo][\'"]\n\s+[\'"]x', 'string join across lines with no space'), |
|
224 | 224 | (r'[^\n]\Z', "no trailing newline"), |
|
225 | 225 | (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"), |
|
226 | 226 | # (r'^\s+[^_ \n][^_. \n]+_[^_\n]+\s*=', |
|
227 | 227 | # "don't use underbars in identifiers"), |
|
228 | 228 | (r'^\s+(self\.)?[A-za-z][a-z0-9]+[A-Z]\w* = ', |
|
229 | 229 | "don't use camelcase in identifiers"), |
|
230 | 230 | (r'^\s*(if|while|def|class|except|try)\s[^[\n]*:\s*[^\\n]#\s]+', |
|
231 | 231 | "linebreak after :"), |
|
232 | 232 | (r'class\s[^( \n]+:', "old-style class, use class foo(object)"), |
|
233 | 233 | (r'class\s[^( \n]+\(\):', |
|
234 | 234 | "class foo() not available in Python 2.4, use class foo(object)"), |
|
235 | 235 | (r'\b(%s)\(' % '|'.join(keyword.kwlist), |
|
236 | 236 | "Python keyword is not a function"), |
|
237 | 237 | (r',]', "unneeded trailing ',' in list"), |
|
238 | 238 | # (r'class\s[A-Z][^\(]*\((?!Exception)', |
|
239 | 239 | # "don't capitalize non-exception classes"), |
|
240 | 240 | # (r'in range\(', "use xrange"), |
|
241 | 241 | # (r'^\s*print\s+', "avoid using print in core and extensions"), |
|
242 | 242 | (r'[\x80-\xff]', "non-ASCII character literal"), |
|
243 | 243 | (r'("\')\.format\(', "str.format() not available in Python 2.4"), |
|
244 | 244 | (r'^\s*with\s+', "with not available in Python 2.4"), |
|
245 | 245 | (r'\.isdisjoint\(', "set.isdisjoint not available in Python 2.4"), |
|
246 | 246 | (r'^\s*except.* as .*:', "except as not available in Python 2.4"), |
|
247 | 247 | (r'^\s*os\.path\.relpath', "relpath not available in Python 2.4"), |
|
248 | 248 | (r'(?<!def)\s+(any|all|format)\(', |
|
249 | 249 | "any/all/format not available in Python 2.4", 'no-py24'), |
|
250 | 250 | (r'(?<!def)\s+(callable)\(', |
|
251 | 251 | "callable not available in Python 3, use getattr(f, '__call__', None)"), |
|
252 | 252 | (r'if\s.*\selse', "if ... else form not available in Python 2.4"), |
|
253 | 253 | (r'^\s*(%s)\s\s' % '|'.join(keyword.kwlist), |
|
254 | 254 | "gratuitous whitespace after Python keyword"), |
|
255 | 255 | (r'([\(\[][ \t]\S)|(\S[ \t][\)\]])', "gratuitous whitespace in () or []"), |
|
256 | 256 | # (r'\s\s=', "gratuitous whitespace before ="), |
|
257 | 257 | (r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S', |
|
258 | 258 | "missing whitespace around operator"), |
|
259 | 259 | (r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\s', |
|
260 | 260 | "missing whitespace around operator"), |
|
261 | 261 | (r'\s(\+=|-=|!=|<>|<=|>=|<<=|>>=|%=)\S', |
|
262 | 262 | "missing whitespace around operator"), |
|
263 | 263 | (r'[^^+=*/!<>&| %-](\s=|=\s)[^= ]', |
|
264 | 264 | "wrong whitespace around ="), |
|
265 | 265 | (r'\([^()]*( =[^=]|[^<>!=]= )', |
|
266 | 266 | "no whitespace around = for named parameters"), |
|
267 | 267 | (r'raise Exception', "don't raise generic exceptions"), |
|
268 | 268 | (r'raise [^,(]+, (\([^\)]+\)|[^,\(\)]+)$', |
|
269 | 269 | "don't use old-style two-argument raise, use Exception(message)"), |
|
270 | 270 | (r' is\s+(not\s+)?["\'0-9-]', "object comparison with literal"), |
|
271 | 271 | (r' [=!]=\s+(True|False|None)', |
|
272 | 272 | "comparison with singleton, use 'is' or 'is not' instead"), |
|
273 | 273 | (r'^\s*(while|if) [01]:', |
|
274 | 274 | "use True/False for constant Boolean expression"), |
|
275 | 275 | (r'(?:(?<!def)\s+|\()hasattr', |
|
276 | 276 | 'hasattr(foo, bar) is broken, use util.safehasattr(foo, bar) instead'), |
|
277 | 277 | (r'opener\([^)]*\).read\(', |
|
278 | 278 | "use opener.read() instead"), |
|
279 | 279 | (r'BaseException', 'not in Python 2.4, use Exception'), |
|
280 | 280 | (r'os\.path\.relpath', 'os.path.relpath is not in Python 2.5'), |
|
281 | 281 | (r'opener\([^)]*\).write\(', |
|
282 | 282 | "use opener.write() instead"), |
|
283 | 283 | (r'[\s\(](open|file)\([^)]*\)\.read\(', |
|
284 | 284 | "use util.readfile() instead"), |
|
285 | 285 | (r'[\s\(](open|file)\([^)]*\)\.write\(', |
|
286 | 286 | "use util.writefile() instead"), |
|
287 | 287 | (r'^[\s\(]*(open(er)?|file)\([^)]*\)', |
|
288 | 288 | "always assign an opened file to a variable, and close it afterwards"), |
|
289 | 289 | (r'[\s\(](open|file)\([^)]*\)\.', |
|
290 | 290 | "always assign an opened file to a variable, and close it afterwards"), |
|
291 | 291 | (r'(?i)descendent', "the proper spelling is descendAnt"), |
|
292 | 292 | (r'\.debug\(\_', "don't mark debug messages for translation"), |
|
293 | 293 | (r'\.strip\(\)\.split\(\)', "no need to strip before splitting"), |
|
294 | 294 | (r'^\s*except\s*:', "naked except clause", r'#.*re-raises'), |
|
295 | 295 | (r':\n( )*( ){1,3}[^ ]', "must indent 4 spaces"), |
|
296 | 296 | (r'ui\.(status|progress|write|note|warn)\([\'\"]x', |
|
297 | 297 | "missing _() in ui message (use () to hide false-positives)"), |
|
298 | 298 | (r'release\(.*wlock, .*lock\)', "wrong lock release order"), |
|
299 | 299 | ], |
|
300 | 300 | # warnings |
|
301 | 301 | [ |
|
302 | 302 | (r'(^| )pp +xxxxqq[ \n][^\n]', "add two newlines after '.. note::'"), |
|
303 | 303 | ] |
|
304 | 304 | ] |
|
305 | 305 | |
|
306 | 306 | pyfilters = [ |
|
307 | 307 | (r"""(?msx)(?P<comment>\#.*?$)| |
|
308 | 308 | ((?P<quote>('''|\"\"\"|(?<!')'(?!')|(?<!")"(?!"))) |
|
309 | 309 | (?P<text>(([^\\]|\\.)*?)) |
|
310 | 310 | (?P=quote))""", reppython), |
|
311 | 311 | ] |
|
312 | 312 | |
|
313 | 313 | txtfilters = [] |
|
314 | 314 | |
|
315 | 315 | txtpats = [ |
|
316 | 316 | [ |
|
317 | 317 | ('\s$', 'trailing whitespace'), |
|
318 | 318 | ('.. note::[ \n][^\n]', 'add two newlines after note::') |
|
319 | 319 | ], |
|
320 | 320 | [] |
|
321 | 321 | ] |
|
322 | 322 | |
|
323 | 323 | cpats = [ |
|
324 | 324 | [ |
|
325 | 325 | (r'//', "don't use //-style comments"), |
|
326 | 326 | (r'^ ', "don't use spaces to indent"), |
|
327 | 327 | (r'\S\t', "don't use tabs except for indent"), |
|
328 | 328 | (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"), |
|
329 | 329 | (r'.{81}', "line too long"), |
|
330 | 330 | (r'(while|if|do|for)\(', "use space after while/if/do/for"), |
|
331 | 331 | (r'return\(', "return is not a function"), |
|
332 | 332 | (r' ;', "no space before ;"), |
|
333 | 333 | (r'[)][{]', "space between ) and {"), |
|
334 | 334 | (r'\w+\* \w+', "use int *foo, not int* foo"), |
|
335 | 335 | (r'\W\([^\)]+\) \w+', "use (int)foo, not (int) foo"), |
|
336 | 336 | (r'\w+ (\+\+|--)', "use foo++, not foo ++"), |
|
337 | 337 | (r'\w,\w', "missing whitespace after ,"), |
|
338 | 338 | (r'^[^#]\w[+/*]\w', "missing whitespace in expression"), |
|
339 | 339 | (r'^#\s+\w', "use #foo, not # foo"), |
|
340 | 340 | (r'[^\n]\Z', "no trailing newline"), |
|
341 | 341 | (r'^\s*#import\b', "use only #include in standard C code"), |
|
342 | 342 | ], |
|
343 | 343 | # warnings |
|
344 | 344 | [] |
|
345 | 345 | ] |
|
346 | 346 | |
|
347 | 347 | cfilters = [ |
|
348 | 348 | (r'(/\*)(((\*(?!/))|[^*])*)\*/', repccomment), |
|
349 | 349 | (r'''(?P<quote>(?<!")")(?P<text>([^"]|\\")+)"(?!")''', repquote), |
|
350 | 350 | (r'''(#\s*include\s+<)([^>]+)>''', repinclude), |
|
351 | 351 | (r'(\()([^)]+\))', repcallspaces), |
|
352 | 352 | ] |
|
353 | 353 | |
|
354 | 354 | inutilpats = [ |
|
355 | 355 | [ |
|
356 | 356 | (r'\bui\.', "don't use ui in util"), |
|
357 | 357 | ], |
|
358 | 358 | # warnings |
|
359 | 359 | [] |
|
360 | 360 | ] |
|
361 | 361 | |
|
362 | 362 | inrevlogpats = [ |
|
363 | 363 | [ |
|
364 | 364 | (r'\brepo\.', "don't use repo in revlog"), |
|
365 | 365 | ], |
|
366 | 366 | # warnings |
|
367 | 367 | [] |
|
368 | 368 | ] |
|
369 | 369 | |
|
370 | 370 | checks = [ |
|
371 | 371 | ('python', r'.*\.(py|cgi)$', pyfilters, pypats), |
|
372 | 372 | ('test script', r'(.*/)?test-[^.~]*$', testfilters, testpats), |
|
373 | 373 | ('c', r'.*\.[ch]$', cfilters, cpats), |
|
374 | 374 | ('unified test', r'.*\.t$', utestfilters, utestpats), |
|
375 | 375 | ('layering violation repo in revlog', r'mercurial/revlog\.py', pyfilters, |
|
376 | 376 | inrevlogpats), |
|
377 | 377 | ('layering violation ui in util', r'mercurial/util\.py', pyfilters, |
|
378 | 378 | inutilpats), |
|
379 | 379 | ('txt', r'.*\.txt$', txtfilters, txtpats), |
|
380 | 380 | ] |
|
381 | 381 | |
|
382 | 382 | def _preparepats(): |
|
383 | 383 | for c in checks: |
|
384 | 384 | failandwarn = c[-1] |
|
385 | 385 | for pats in failandwarn: |
|
386 | 386 | for i, pseq in enumerate(pats): |
|
387 | 387 | # fix-up regexes for multi-line searches |
|
388 | 388 | p = pseq[0] |
|
389 | 389 | # \s doesn't match \n |
|
390 | 390 | p = re.sub(r'(?<!\\)\\s', r'[ \\t]', p) |
|
391 | 391 | # [^...] doesn't match newline |
|
392 | 392 | p = re.sub(r'(?<!\\)\[\^', r'[^\\n', p) |
|
393 | 393 | |
|
394 | 394 | pats[i] = (re.compile(p, re.MULTILINE),) + pseq[1:] |
|
395 | 395 | filters = c[2] |
|
396 | 396 | for i, flt in enumerate(filters): |
|
397 | 397 | filters[i] = re.compile(flt[0]), flt[1] |
|
398 | 398 | _preparepats() |
|
399 | 399 | |
|
400 | 400 | class norepeatlogger(object): |
|
401 | 401 | def __init__(self): |
|
402 | 402 | self._lastseen = None |
|
403 | 403 | |
|
404 | 404 | def log(self, fname, lineno, line, msg, blame): |
|
405 | 405 | """print error related a to given line of a given file. |
|
406 | 406 | |
|
407 | 407 | The faulty line will also be printed but only once in the case |
|
408 | 408 | of multiple errors. |
|
409 | 409 | |
|
410 | 410 | :fname: filename |
|
411 | 411 | :lineno: line number |
|
412 | 412 | :line: actual content of the line |
|
413 | 413 | :msg: error message |
|
414 | 414 | """ |
|
415 | 415 | msgid = fname, lineno, line |
|
416 | 416 | if msgid != self._lastseen: |
|
417 | 417 | if blame: |
|
418 | 418 | print "%s:%d (%s):" % (fname, lineno, blame) |
|
419 | 419 | else: |
|
420 | 420 | print "%s:%d:" % (fname, lineno) |
|
421 | 421 | print " > %s" % line |
|
422 | 422 | self._lastseen = msgid |
|
423 | 423 | print " " + msg |
|
424 | 424 | |
|
425 | 425 | _defaultlogger = norepeatlogger() |
|
426 | 426 | |
|
427 | 427 | def getblame(f): |
|
428 | 428 | lines = [] |
|
429 | 429 | for l in os.popen('hg annotate -un %s' % f): |
|
430 | 430 | start, line = l.split(':', 1) |
|
431 | 431 | user, rev = start.split() |
|
432 | 432 | lines.append((line[1:-1], user, rev)) |
|
433 | 433 | return lines |
|
434 | 434 | |
|
435 | 435 | def checkfile(f, logfunc=_defaultlogger.log, maxerr=None, warnings=False, |
|
436 | 436 | blame=False, debug=False, lineno=True): |
|
437 | 437 | """checks style and portability of a given file |
|
438 | 438 | |
|
439 | 439 | :f: filepath |
|
440 | 440 | :logfunc: function used to report error |
|
441 | 441 | logfunc(filename, linenumber, linecontent, errormessage) |
|
442 | 442 | :maxerr: number of error to display before aborting. |
|
443 | 443 | Set to false (default) to report all errors |
|
444 | 444 | |
|
445 | 445 | return True if no error is found, False otherwise. |
|
446 | 446 | """ |
|
447 | 447 | blamecache = None |
|
448 | 448 | result = True |
|
449 | 449 | for name, match, filters, pats in checks: |
|
450 | 450 | if debug: |
|
451 | 451 | print name, f |
|
452 | 452 | fc = 0 |
|
453 | 453 | if not re.match(match, f): |
|
454 | 454 | if debug: |
|
455 | 455 | print "Skipping %s for %s it doesn't match %s" % ( |
|
456 | 456 | name, match, f) |
|
457 | 457 | continue |
|
458 | 458 | try: |
|
459 | 459 | fp = open(f) |
|
460 | 460 | except IOError, e: |
|
461 | 461 | print "Skipping %s, %s" % (f, str(e).split(':', 1)[0]) |
|
462 | 462 | continue |
|
463 | 463 | pre = post = fp.read() |
|
464 | 464 | fp.close() |
|
465 | 465 | if "no-" "check-code" in pre: |
|
466 | 466 | print "Skipping %s it has no-" "check-code" % f |
|
467 | 467 | return "Skip" # skip checking this file |
|
468 | 468 | for p, r in filters: |
|
469 | 469 | post = re.sub(p, r, post) |
|
470 | 470 | nerrs = len(pats[0]) # nerr elements are errors |
|
471 | 471 | if warnings: |
|
472 | 472 | pats = pats[0] + pats[1] |
|
473 | 473 | else: |
|
474 | 474 | pats = pats[0] |
|
475 | 475 | # print post # uncomment to show filtered version |
|
476 | 476 | |
|
477 | 477 | if debug: |
|
478 | 478 | print "Checking %s for %s" % (name, f) |
|
479 | 479 | |
|
480 | 480 | prelines = None |
|
481 | 481 | errors = [] |
|
482 | 482 | for i, pat in enumerate(pats): |
|
483 | 483 | if len(pat) == 3: |
|
484 | 484 | p, msg, ignore = pat |
|
485 | 485 | else: |
|
486 | 486 | p, msg = pat |
|
487 | 487 | ignore = None |
|
488 | 488 | if i >= nerrs: |
|
489 | 489 | msg = "warning: " + msg |
|
490 | 490 | |
|
491 | 491 | pos = 0 |
|
492 | 492 | n = 0 |
|
493 | 493 | for m in p.finditer(post): |
|
494 | 494 | if prelines is None: |
|
495 | 495 | prelines = pre.splitlines() |
|
496 | 496 | postlines = post.splitlines(True) |
|
497 | 497 | |
|
498 | 498 | start = m.start() |
|
499 | 499 | while n < len(postlines): |
|
500 | 500 | step = len(postlines[n]) |
|
501 | 501 | if pos + step > start: |
|
502 | 502 | break |
|
503 | 503 | pos += step |
|
504 | 504 | n += 1 |
|
505 | 505 | l = prelines[n] |
|
506 | 506 | |
|
507 | 507 | if ignore and re.search(ignore, l, re.MULTILINE): |
|
508 | 508 | if debug: |
|
509 | 509 | print "Skipping %s for %s:%s (ignore pattern)" % ( |
|
510 | 510 | name, f, n) |
|
511 | 511 | continue |
|
512 | 512 | bd = "" |
|
513 | 513 | if blame: |
|
514 | 514 | bd = 'working directory' |
|
515 | 515 | if not blamecache: |
|
516 | 516 | blamecache = getblame(f) |
|
517 | 517 | if n < len(blamecache): |
|
518 | 518 | bl, bu, br = blamecache[n] |
|
519 | 519 | if bl == l: |
|
520 | 520 | bd = '%s@%s' % (bu, br) |
|
521 | 521 | |
|
522 | 522 | errors.append((f, lineno and n + 1, l, msg, bd)) |
|
523 | 523 | result = False |
|
524 | 524 | |
|
525 | 525 | errors.sort() |
|
526 | 526 | for e in errors: |
|
527 | 527 | logfunc(*e) |
|
528 | 528 | fc += 1 |
|
529 | 529 | if maxerr and fc >= maxerr: |
|
530 | 530 | print " (too many errors, giving up)" |
|
531 | 531 | break |
|
532 | 532 | |
|
533 | 533 | return result |
|
534 | 534 | |
|
535 | 535 | if __name__ == "__main__": |
|
536 | 536 | parser = optparse.OptionParser("%prog [options] [files]") |
|
537 | 537 | parser.add_option("-w", "--warnings", action="store_true", |
|
538 | 538 | help="include warning-level checks") |
|
539 | 539 | parser.add_option("-p", "--per-file", type="int", |
|
540 | 540 | help="max warnings per file") |
|
541 | 541 | parser.add_option("-b", "--blame", action="store_true", |
|
542 | 542 | help="use annotate to generate blame info") |
|
543 | 543 | parser.add_option("", "--debug", action="store_true", |
|
544 | 544 | help="show debug information") |
|
545 | 545 | parser.add_option("", "--nolineno", action="store_false", |
|
546 | 546 | dest='lineno', help="don't show line numbers") |
|
547 | 547 | |
|
548 | 548 | parser.set_defaults(per_file=15, warnings=False, blame=False, debug=False, |
|
549 | 549 | lineno=True) |
|
550 | 550 | (options, args) = parser.parse_args() |
|
551 | 551 | |
|
552 | 552 | if len(args) == 0: |
|
553 | 553 | check = glob.glob("*") |
|
554 | 554 | else: |
|
555 | 555 | check = args |
|
556 | 556 | |
|
557 | 557 | ret = 0 |
|
558 | 558 | for f in check: |
|
559 | 559 | if not checkfile(f, maxerr=options.per_file, warnings=options.warnings, |
|
560 | 560 | blame=options.blame, debug=options.debug, |
|
561 | 561 | lineno=options.lineno): |
|
562 | 562 | ret = 1 |
|
563 | 563 | sys.exit(ret) |
@@ -1,350 +1,350 b'' | |||
|
1 | 1 | """automatically manage newlines in repository files |
|
2 | 2 | |
|
3 | 3 | This extension allows you to manage the type of line endings (CRLF or |
|
4 | 4 | LF) that are used in the repository and in the local working |
|
5 | 5 | directory. That way you can get CRLF line endings on Windows and LF on |
|
6 | 6 | Unix/Mac, thereby letting everybody use their OS native line endings. |
|
7 | 7 | |
|
8 | 8 | The extension reads its configuration from a versioned ``.hgeol`` |
|
9 | 9 | configuration file found in the root of the working copy. The |
|
10 | 10 | ``.hgeol`` file use the same syntax as all other Mercurial |
|
11 | 11 | configuration files. It uses two sections, ``[patterns]`` and |
|
12 | 12 | ``[repository]``. |
|
13 | 13 | |
|
14 | 14 | The ``[patterns]`` section specifies how line endings should be |
|
15 | 15 | converted between the working copy and the repository. The format is |
|
16 | 16 | specified by a file pattern. The first match is used, so put more |
|
17 | 17 | specific patterns first. The available line endings are ``LF``, |
|
18 | 18 | ``CRLF``, and ``BIN``. |
|
19 | 19 | |
|
20 | 20 | Files with the declared format of ``CRLF`` or ``LF`` are always |
|
21 | 21 | checked out and stored in the repository in that format and files |
|
22 | 22 | declared to be binary (``BIN``) are left unchanged. Additionally, |
|
23 | 23 | ``native`` is an alias for checking out in the platform's default line |
|
24 | 24 | ending: ``LF`` on Unix (including Mac OS X) and ``CRLF`` on |
|
25 | 25 | Windows. Note that ``BIN`` (do nothing to line endings) is Mercurial's |
|
26 | 26 | default behaviour; it is only needed if you need to override a later, |
|
27 | 27 | more general pattern. |
|
28 | 28 | |
|
29 | 29 | The optional ``[repository]`` section specifies the line endings to |
|
30 | 30 | use for files stored in the repository. It has a single setting, |
|
31 | 31 | ``native``, which determines the storage line endings for files |
|
32 | 32 | declared as ``native`` in the ``[patterns]`` section. It can be set to |
|
33 | 33 | ``LF`` or ``CRLF``. The default is ``LF``. For example, this means |
|
34 | 34 | that on Windows, files configured as ``native`` (``CRLF`` by default) |
|
35 | 35 | will be converted to ``LF`` when stored in the repository. Files |
|
36 | 36 | declared as ``LF``, ``CRLF``, or ``BIN`` in the ``[patterns]`` section |
|
37 | 37 | are always stored as-is in the repository. |
|
38 | 38 | |
|
39 | 39 | Example versioned ``.hgeol`` file:: |
|
40 | 40 | |
|
41 | 41 | [patterns] |
|
42 | 42 | **.py = native |
|
43 | 43 | **.vcproj = CRLF |
|
44 | 44 | **.txt = native |
|
45 | 45 | Makefile = LF |
|
46 | 46 | **.jpg = BIN |
|
47 | 47 | |
|
48 | 48 | [repository] |
|
49 | 49 | native = LF |
|
50 | 50 | |
|
51 | 51 | .. note:: |
|
52 | 52 | |
|
53 | 53 | The rules will first apply when files are touched in the working |
|
54 | 54 | copy, e.g. by updating to null and back to tip to touch all files. |
|
55 | 55 | |
|
56 | 56 | The extension uses an optional ``[eol]`` section read from both the |
|
57 | 57 | normal Mercurial configuration files and the ``.hgeol`` file, with the |
|
58 | 58 | latter overriding the former. You can use that section to control the |
|
59 | 59 | overall behavior. There are three settings: |
|
60 | 60 | |
|
61 | 61 | - ``eol.native`` (default ``os.linesep``) can be set to ``LF`` or |
|
62 | 62 | ``CRLF`` to override the default interpretation of ``native`` for |
|
63 | 63 | checkout. This can be used with :hg:`archive` on Unix, say, to |
|
64 | 64 | generate an archive where files have line endings for Windows. |
|
65 | 65 | |
|
66 | 66 | - ``eol.only-consistent`` (default True) can be set to False to make |
|
67 | 67 | the extension convert files with inconsistent EOLs. Inconsistent |
|
68 | 68 | means that there is both ``CRLF`` and ``LF`` present in the file. |
|
69 | 69 | Such files are normally not touched under the assumption that they |
|
70 | 70 | have mixed EOLs on purpose. |
|
71 | 71 | |
|
72 | 72 | - ``eol.fix-trailing-newline`` (default False) can be set to True to |
|
73 | 73 | ensure that converted files end with a EOL character (either ``\\n`` |
|
74 | 74 | or ``\\r\\n`` as per the configured patterns). |
|
75 | 75 | |
|
76 | 76 | The extension provides ``cleverencode:`` and ``cleverdecode:`` filters |
|
77 | 77 | like the deprecated win32text extension does. This means that you can |
|
78 | 78 | disable win32text and enable eol and your filters will still work. You |
|
79 | 79 | only need to these filters until you have prepared a ``.hgeol`` file. |
|
80 | 80 | |
|
81 | 81 | The ``win32text.forbid*`` hooks provided by the win32text extension |
|
82 | 82 | have been unified into a single hook named ``eol.checkheadshook``. The |
|
83 | 83 | hook will lookup the expected line endings from the ``.hgeol`` file, |
|
84 | 84 | which means you must migrate to a ``.hgeol`` file first before using |
|
85 | 85 | the hook. ``eol.checkheadshook`` only checks heads, intermediate |
|
86 | 86 | invalid revisions will be pushed. To forbid them completely, use the |
|
87 | 87 | ``eol.checkallhook`` hook. These hooks are best used as |
|
88 | 88 | ``pretxnchangegroup`` hooks. |
|
89 | 89 | |
|
90 | 90 | See :hg:`help patterns` for more information about the glob patterns |
|
91 | 91 | used. |
|
92 | 92 | """ |
|
93 | 93 | |
|
94 | 94 | from mercurial.i18n import _ |
|
95 | 95 | from mercurial import util, config, extensions, match, error |
|
96 | 96 | import re, os |
|
97 | 97 | |
|
98 | 98 | testedwith = 'internal' |
|
99 | 99 | |
|
100 | 100 | # Matches a lone LF, i.e., one that is not part of CRLF. |
|
101 | 101 | singlelf = re.compile('(^|[^\r])\n') |
|
102 | 102 | # Matches a single EOL which can either be a CRLF where repeated CR |
|
103 | 103 | # are removed or a LF. We do not care about old Macintosh files, so a |
|
104 | 104 | # stray CR is an error. |
|
105 | 105 | eolre = re.compile('\r*\n') |
|
106 | 106 | |
|
107 | 107 | |
|
108 | 108 | def inconsistenteol(data): |
|
109 | 109 | return '\r\n' in data and singlelf.search(data) |
|
110 | 110 | |
|
111 | 111 | def tolf(s, params, ui, **kwargs): |
|
112 | 112 | """Filter to convert to LF EOLs.""" |
|
113 | 113 | if util.binary(s): |
|
114 | 114 | return s |
|
115 | 115 | if ui.configbool('eol', 'only-consistent', True) and inconsistenteol(s): |
|
116 | 116 | return s |
|
117 | 117 | if (ui.configbool('eol', 'fix-trailing-newline', False) |
|
118 | 118 | and s and s[-1] != '\n'): |
|
119 | 119 | s = s + '\n' |
|
120 | 120 | return eolre.sub('\n', s) |
|
121 | 121 | |
|
122 | 122 | def tocrlf(s, params, ui, **kwargs): |
|
123 | 123 | """Filter to convert to CRLF EOLs.""" |
|
124 | 124 | if util.binary(s): |
|
125 | 125 | return s |
|
126 | 126 | if ui.configbool('eol', 'only-consistent', True) and inconsistenteol(s): |
|
127 | 127 | return s |
|
128 | 128 | if (ui.configbool('eol', 'fix-trailing-newline', False) |
|
129 | 129 | and s and s[-1] != '\n'): |
|
130 | 130 | s = s + '\n' |
|
131 | 131 | return eolre.sub('\r\n', s) |
|
132 | 132 | |
|
133 | 133 | def isbinary(s, params): |
|
134 | 134 | """Filter to do nothing with the file.""" |
|
135 | 135 | return s |
|
136 | 136 | |
|
137 | 137 | filters = { |
|
138 | 138 | 'to-lf': tolf, |
|
139 | 139 | 'to-crlf': tocrlf, |
|
140 | 140 | 'is-binary': isbinary, |
|
141 | 141 | # The following provide backwards compatibility with win32text |
|
142 | 142 | 'cleverencode:': tolf, |
|
143 | 143 | 'cleverdecode:': tocrlf |
|
144 | 144 | } |
|
145 | 145 | |
|
146 | 146 | class eolfile(object): |
|
147 | 147 | def __init__(self, ui, root, data): |
|
148 | 148 | self._decode = {'LF': 'to-lf', 'CRLF': 'to-crlf', 'BIN': 'is-binary'} |
|
149 | 149 | self._encode = {'LF': 'to-lf', 'CRLF': 'to-crlf', 'BIN': 'is-binary'} |
|
150 | 150 | |
|
151 | 151 | self.cfg = config.config() |
|
152 | 152 | # Our files should not be touched. The pattern must be |
|
153 | 153 | # inserted first override a '** = native' pattern. |
|
154 | 154 | self.cfg.set('patterns', '.hg*', 'BIN', 'eol') |
|
155 | 155 | # We can then parse the user's patterns. |
|
156 | 156 | self.cfg.parse('.hgeol', data) |
|
157 | 157 | |
|
158 | 158 | isrepolf = self.cfg.get('repository', 'native') != 'CRLF' |
|
159 | 159 | self._encode['NATIVE'] = isrepolf and 'to-lf' or 'to-crlf' |
|
160 | 160 | iswdlf = ui.config('eol', 'native', os.linesep) in ('LF', '\n') |
|
161 | 161 | self._decode['NATIVE'] = iswdlf and 'to-lf' or 'to-crlf' |
|
162 | 162 | |
|
163 | 163 | include = [] |
|
164 | 164 | exclude = [] |
|
165 | 165 | for pattern, style in self.cfg.items('patterns'): |
|
166 | 166 | key = style.upper() |
|
167 | 167 | if key == 'BIN': |
|
168 | 168 | exclude.append(pattern) |
|
169 | 169 | else: |
|
170 | 170 | include.append(pattern) |
|
171 | 171 | # This will match the files for which we need to care |
|
172 | 172 | # about inconsistent newlines. |
|
173 | 173 | self.match = match.match(root, '', [], include, exclude) |
|
174 | 174 | |
|
175 | 175 | def copytoui(self, ui): |
|
176 | 176 | for pattern, style in self.cfg.items('patterns'): |
|
177 | 177 | key = style.upper() |
|
178 | 178 | try: |
|
179 | 179 | ui.setconfig('decode', pattern, self._decode[key], 'eol') |
|
180 | 180 | ui.setconfig('encode', pattern, self._encode[key], 'eol') |
|
181 | 181 | except KeyError: |
|
182 | 182 | ui.warn(_("ignoring unknown EOL style '%s' from %s\n") |
|
183 | 183 | % (style, self.cfg.source('patterns', pattern))) |
|
184 | 184 | # eol.only-consistent can be specified in ~/.hgrc or .hgeol |
|
185 | 185 | for k, v in self.cfg.items('eol'): |
|
186 | 186 | ui.setconfig('eol', k, v, 'eol') |
|
187 | 187 | |
|
188 | 188 | def checkrev(self, repo, ctx, files): |
|
189 | 189 | failed = [] |
|
190 | 190 | for f in (files or ctx.files()): |
|
191 | 191 | if f not in ctx: |
|
192 | 192 | continue |
|
193 | 193 | for pattern, style in self.cfg.items('patterns'): |
|
194 | 194 | if not match.match(repo.root, '', [pattern])(f): |
|
195 | 195 | continue |
|
196 | 196 | target = self._encode[style.upper()] |
|
197 | 197 | data = ctx[f].data() |
|
198 | 198 | if (target == "to-lf" and "\r\n" in data |
|
199 | 199 | or target == "to-crlf" and singlelf.search(data)): |
|
200 | 200 | failed.append((str(ctx), target, f)) |
|
201 | 201 | break |
|
202 | 202 | return failed |
|
203 | 203 | |
|
204 | 204 | def parseeol(ui, repo, nodes): |
|
205 | 205 | try: |
|
206 | 206 | for node in nodes: |
|
207 | 207 | try: |
|
208 | 208 | if node is None: |
|
209 | 209 | # Cannot use workingctx.data() since it would load |
|
210 | 210 | # and cache the filters before we configure them. |
|
211 | 211 | data = repo.wfile('.hgeol').read() |
|
212 | 212 | else: |
|
213 | 213 | data = repo[node]['.hgeol'].data() |
|
214 | 214 | return eolfile(ui, repo.root, data) |
|
215 | 215 | except (IOError, LookupError): |
|
216 | 216 | pass |
|
217 | 217 | except error.ParseError, inst: |
|
218 | 218 | ui.warn(_("warning: ignoring .hgeol file due to parse error " |
|
219 | 219 | "at %s: %s\n") % (inst.args[1], inst.args[0])) |
|
220 | 220 | return None |
|
221 | 221 | |
|
222 | 222 | def _checkhook(ui, repo, node, headsonly): |
|
223 | 223 | # Get revisions to check and touched files at the same time |
|
224 | 224 | files = set() |
|
225 | 225 | revs = set() |
|
226 | 226 | for rev in xrange(repo[node].rev(), len(repo)): |
|
227 | 227 | revs.add(rev) |
|
228 | 228 | if headsonly: |
|
229 | 229 | ctx = repo[rev] |
|
230 | 230 | files.update(ctx.files()) |
|
231 | 231 | for pctx in ctx.parents(): |
|
232 | 232 | revs.discard(pctx.rev()) |
|
233 | 233 | failed = [] |
|
234 | 234 | for rev in revs: |
|
235 | 235 | ctx = repo[rev] |
|
236 | 236 | eol = parseeol(ui, repo, [ctx.node()]) |
|
237 | 237 | if eol: |
|
238 | 238 | failed.extend(eol.checkrev(repo, ctx, files)) |
|
239 | 239 | |
|
240 | 240 | if failed: |
|
241 | 241 | eols = {'to-lf': 'CRLF', 'to-crlf': 'LF'} |
|
242 | 242 | msgs = [] |
|
243 | 243 | for node, target, f in failed: |
|
244 | 244 | msgs.append(_(" %s in %s should not have %s line endings") % |
|
245 | 245 | (f, node, eols[target])) |
|
246 | 246 | raise util.Abort(_("end-of-line check failed:\n") + "\n".join(msgs)) |
|
247 | 247 | |
|
248 | 248 | def checkallhook(ui, repo, node, hooktype, **kwargs): |
|
249 | 249 | """verify that files have expected EOLs""" |
|
250 | 250 | _checkhook(ui, repo, node, False) |
|
251 | 251 | |
|
252 | 252 | def checkheadshook(ui, repo, node, hooktype, **kwargs): |
|
253 | 253 | """verify that files have expected EOLs""" |
|
254 | 254 | _checkhook(ui, repo, node, True) |
|
255 | 255 | |
|
256 | 256 | # "checkheadshook" used to be called "hook" |
|
257 | 257 | hook = checkheadshook |
|
258 | 258 | |
|
259 | 259 | def preupdate(ui, repo, hooktype, parent1, parent2): |
|
260 | 260 | repo.loadeol([parent1]) |
|
261 | 261 | return False |
|
262 | 262 | |
|
263 | 263 | def uisetup(ui): |
|
264 | 264 | ui.setconfig('hooks', 'preupdate.eol', preupdate, 'eol') |
|
265 | 265 | |
|
266 | 266 | def extsetup(ui): |
|
267 | 267 | try: |
|
268 | 268 | extensions.find('win32text') |
|
269 | 269 | ui.warn(_("the eol extension is incompatible with the " |
|
270 | 270 | "win32text extension\n")) |
|
271 | 271 | except KeyError: |
|
272 | 272 | pass |
|
273 | 273 | |
|
274 | 274 | |
|
275 | 275 | def reposetup(ui, repo): |
|
276 | 276 | uisetup(repo.ui) |
|
277 | 277 | |
|
278 | 278 | if not repo.local(): |
|
279 | 279 | return |
|
280 | 280 | for name, fn in filters.iteritems(): |
|
281 | 281 | repo.adddatafilter(name, fn) |
|
282 | 282 | |
|
283 | 283 | ui.setconfig('patch', 'eol', 'auto', 'eol') |
|
284 | 284 | |
|
285 | 285 | class eolrepo(repo.__class__): |
|
286 | 286 | |
|
287 | 287 | def loadeol(self, nodes): |
|
288 | 288 | eol = parseeol(self.ui, self, nodes) |
|
289 | 289 | if eol is None: |
|
290 | 290 | return None |
|
291 | 291 | eol.copytoui(self.ui) |
|
292 | 292 | return eol.match |
|
293 | 293 | |
|
294 | 294 | def _hgcleardirstate(self): |
|
295 | 295 | self._eolfile = self.loadeol([None, 'tip']) |
|
296 | 296 | if not self._eolfile: |
|
297 | 297 | self._eolfile = util.never |
|
298 | 298 | return |
|
299 | 299 | |
|
300 | 300 | try: |
|
301 | 301 | cachemtime = os.path.getmtime(self.join("eol.cache")) |
|
302 | 302 | except OSError: |
|
303 | 303 | cachemtime = 0 |
|
304 | 304 | |
|
305 | 305 | try: |
|
306 | 306 | eolmtime = os.path.getmtime(self.wjoin(".hgeol")) |
|
307 | 307 | except OSError: |
|
308 | 308 | eolmtime = 0 |
|
309 | 309 | |
|
310 | 310 | if eolmtime > cachemtime: |
|
311 | 311 | self.ui.debug("eol: detected change in .hgeol\n") |
|
312 | 312 | wlock = None |
|
313 | 313 | try: |
|
314 | 314 | wlock = self.wlock() |
|
315 | 315 | for f in self.dirstate: |
|
316 | 316 | if self.dirstate[f] == 'n': |
|
317 | 317 | # all normal files need to be looked at |
|
318 | 318 | # again since the new .hgeol file might no |
|
319 | 319 | # longer match a file it matched before |
|
320 | 320 | self.dirstate.normallookup(f) |
|
321 | 321 | # Create or touch the cache to update mtime |
|
322 | 322 | self.opener("eol.cache", "w").close() |
|
323 | 323 | wlock.release() |
|
324 | 324 | except error.LockUnavailable: |
|
325 | 325 | # If we cannot lock the repository and clear the |
|
326 | 326 | # dirstate, then a commit might not see all files |
|
327 | 327 | # as modified. But if we cannot lock the |
|
328 | 328 | # repository, then we can also not make a commit, |
|
329 | 329 | # so ignore the error. |
|
330 | 330 | pass |
|
331 | 331 | |
|
332 | 332 | def commitctx(self, ctx, error=False): |
|
333 | 333 | for f in sorted(ctx.added() + ctx.modified()): |
|
334 | 334 | if not self._eolfile(f): |
|
335 | 335 | continue |
|
336 | 336 | try: |
|
337 | 337 | data = ctx[f].data() |
|
338 | 338 | except IOError: |
|
339 | 339 | continue |
|
340 | 340 | if util.binary(data): |
|
341 | 341 | # We should not abort here, since the user should |
|
342 | 342 | # be able to say "** = native" to automatically |
|
343 | 343 | # have all non-binary files taken care of. |
|
344 | 344 | continue |
|
345 | 345 | if inconsistenteol(data): |
|
346 | 346 | raise util.Abort(_("inconsistent newline style " |
|
347 |
"in %s\n" % f) |
|
|
347 | "in %s\n") % f) | |
|
348 | 348 | return super(eolrepo, self).commitctx(ctx, error) |
|
349 | 349 | repo.__class__ = eolrepo |
|
350 | 350 | repo._hgcleardirstate() |
@@ -1,582 +1,582 b'' | |||
|
1 | 1 | # Copyright 2009-2010 Gregory P. Ward |
|
2 | 2 | # Copyright 2009-2010 Intelerad Medical Systems Incorporated |
|
3 | 3 | # Copyright 2010-2011 Fog Creek Software |
|
4 | 4 | # Copyright 2010-2011 Unity Technologies |
|
5 | 5 | # |
|
6 | 6 | # This software may be used and distributed according to the terms of the |
|
7 | 7 | # GNU General Public License version 2 or any later version. |
|
8 | 8 | |
|
9 | 9 | '''High-level command function for lfconvert, plus the cmdtable.''' |
|
10 | 10 | |
|
11 | 11 | import os, errno |
|
12 | 12 | import shutil |
|
13 | 13 | |
|
14 | 14 | from mercurial import util, match as match_, hg, node, context, error, \ |
|
15 | 15 | cmdutil, scmutil, commands |
|
16 | 16 | from mercurial.i18n import _ |
|
17 | 17 | from mercurial.lock import release |
|
18 | 18 | |
|
19 | 19 | import lfutil |
|
20 | 20 | import basestore |
|
21 | 21 | |
|
22 | 22 | # -- Commands ---------------------------------------------------------- |
|
23 | 23 | |
|
24 | 24 | def lfconvert(ui, src, dest, *pats, **opts): |
|
25 | 25 | '''convert a normal repository to a largefiles repository |
|
26 | 26 | |
|
27 | 27 | Convert repository SOURCE to a new repository DEST, identical to |
|
28 | 28 | SOURCE except that certain files will be converted as largefiles: |
|
29 | 29 | specifically, any file that matches any PATTERN *or* whose size is |
|
30 | 30 | above the minimum size threshold is converted as a largefile. The |
|
31 | 31 | size used to determine whether or not to track a file as a |
|
32 | 32 | largefile is the size of the first version of the file. The |
|
33 | 33 | minimum size can be specified either with --size or in |
|
34 | 34 | configuration as ``largefiles.size``. |
|
35 | 35 | |
|
36 | 36 | After running this command you will need to make sure that |
|
37 | 37 | largefiles is enabled anywhere you intend to push the new |
|
38 | 38 | repository. |
|
39 | 39 | |
|
40 | 40 | Use --to-normal to convert largefiles back to normal files; after |
|
41 | 41 | this, the DEST repository can be used without largefiles at all.''' |
|
42 | 42 | |
|
43 | 43 | if opts['to_normal']: |
|
44 | 44 | tolfile = False |
|
45 | 45 | else: |
|
46 | 46 | tolfile = True |
|
47 | 47 | size = lfutil.getminsize(ui, True, opts.get('size'), default=None) |
|
48 | 48 | |
|
49 | 49 | if not hg.islocal(src): |
|
50 | 50 | raise util.Abort(_('%s is not a local Mercurial repo') % src) |
|
51 | 51 | if not hg.islocal(dest): |
|
52 | 52 | raise util.Abort(_('%s is not a local Mercurial repo') % dest) |
|
53 | 53 | |
|
54 | 54 | rsrc = hg.repository(ui, src) |
|
55 | 55 | ui.status(_('initializing destination %s\n') % dest) |
|
56 | 56 | rdst = hg.repository(ui, dest, create=True) |
|
57 | 57 | |
|
58 | 58 | success = False |
|
59 | 59 | dstwlock = dstlock = None |
|
60 | 60 | try: |
|
61 | 61 | # Lock destination to prevent modification while it is converted to. |
|
62 | 62 | # Don't need to lock src because we are just reading from its history |
|
63 | 63 | # which can't change. |
|
64 | 64 | dstwlock = rdst.wlock() |
|
65 | 65 | dstlock = rdst.lock() |
|
66 | 66 | |
|
67 | 67 | # Get a list of all changesets in the source. The easy way to do this |
|
68 | 68 | # is to simply walk the changelog, using changelog.nodesbetween(). |
|
69 | 69 | # Take a look at mercurial/revlog.py:639 for more details. |
|
70 | 70 | # Use a generator instead of a list to decrease memory usage |
|
71 | 71 | ctxs = (rsrc[ctx] for ctx in rsrc.changelog.nodesbetween(None, |
|
72 | 72 | rsrc.heads())[0]) |
|
73 | 73 | revmap = {node.nullid: node.nullid} |
|
74 | 74 | if tolfile: |
|
75 | 75 | lfiles = set() |
|
76 | 76 | normalfiles = set() |
|
77 | 77 | if not pats: |
|
78 | 78 | pats = ui.configlist(lfutil.longname, 'patterns', default=[]) |
|
79 | 79 | if pats: |
|
80 | 80 | matcher = match_.match(rsrc.root, '', list(pats)) |
|
81 | 81 | else: |
|
82 | 82 | matcher = None |
|
83 | 83 | |
|
84 | 84 | lfiletohash = {} |
|
85 | 85 | for ctx in ctxs: |
|
86 | 86 | ui.progress(_('converting revisions'), ctx.rev(), |
|
87 | 87 | unit=_('revision'), total=rsrc['tip'].rev()) |
|
88 | 88 | _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, |
|
89 | 89 | lfiles, normalfiles, matcher, size, lfiletohash) |
|
90 | 90 | ui.progress(_('converting revisions'), None) |
|
91 | 91 | |
|
92 | 92 | if os.path.exists(rdst.wjoin(lfutil.shortname)): |
|
93 | 93 | shutil.rmtree(rdst.wjoin(lfutil.shortname)) |
|
94 | 94 | |
|
95 | 95 | for f in lfiletohash.keys(): |
|
96 | 96 | if os.path.isfile(rdst.wjoin(f)): |
|
97 | 97 | os.unlink(rdst.wjoin(f)) |
|
98 | 98 | try: |
|
99 | 99 | os.removedirs(os.path.dirname(rdst.wjoin(f))) |
|
100 | 100 | except OSError: |
|
101 | 101 | pass |
|
102 | 102 | |
|
103 | 103 | # If there were any files converted to largefiles, add largefiles |
|
104 | 104 | # to the destination repository's requirements. |
|
105 | 105 | if lfiles: |
|
106 | 106 | rdst.requirements.add('largefiles') |
|
107 | 107 | rdst._writerequirements() |
|
108 | 108 | else: |
|
109 | 109 | for ctx in ctxs: |
|
110 | 110 | ui.progress(_('converting revisions'), ctx.rev(), |
|
111 | 111 | unit=_('revision'), total=rsrc['tip'].rev()) |
|
112 | 112 | _addchangeset(ui, rsrc, rdst, ctx, revmap) |
|
113 | 113 | |
|
114 | 114 | ui.progress(_('converting revisions'), None) |
|
115 | 115 | success = True |
|
116 | 116 | finally: |
|
117 | 117 | rdst.dirstate.clear() |
|
118 | 118 | release(dstlock, dstwlock) |
|
119 | 119 | if not success: |
|
120 | 120 | # we failed, remove the new directory |
|
121 | 121 | shutil.rmtree(rdst.root) |
|
122 | 122 | |
|
123 | 123 | def _addchangeset(ui, rsrc, rdst, ctx, revmap): |
|
124 | 124 | # Convert src parents to dst parents |
|
125 | 125 | parents = _convertparents(ctx, revmap) |
|
126 | 126 | |
|
127 | 127 | # Generate list of changed files |
|
128 | 128 | files = _getchangedfiles(ctx, parents) |
|
129 | 129 | |
|
130 | 130 | def getfilectx(repo, memctx, f): |
|
131 | 131 | if lfutil.standin(f) in files: |
|
132 | 132 | # if the file isn't in the manifest then it was removed |
|
133 | 133 | # or renamed, raise IOError to indicate this |
|
134 | 134 | try: |
|
135 | 135 | fctx = ctx.filectx(lfutil.standin(f)) |
|
136 | 136 | except error.LookupError: |
|
137 | 137 | raise IOError |
|
138 | 138 | renamed = fctx.renamed() |
|
139 | 139 | if renamed: |
|
140 | 140 | renamed = lfutil.splitstandin(renamed[0]) |
|
141 | 141 | |
|
142 | 142 | hash = fctx.data().strip() |
|
143 | 143 | path = lfutil.findfile(rsrc, hash) |
|
144 | 144 | |
|
145 | 145 | # If one file is missing, likely all files from this rev are |
|
146 | 146 | if path is None: |
|
147 | 147 | cachelfiles(ui, rsrc, ctx.node()) |
|
148 | 148 | path = lfutil.findfile(rsrc, hash) |
|
149 | 149 | |
|
150 | 150 | if path is None: |
|
151 | 151 | raise util.Abort( |
|
152 | 152 | _("missing largefile \'%s\' from revision %s") |
|
153 | 153 | % (f, node.hex(ctx.node()))) |
|
154 | 154 | |
|
155 | 155 | data = '' |
|
156 | 156 | fd = None |
|
157 | 157 | try: |
|
158 | 158 | fd = open(path, 'rb') |
|
159 | 159 | data = fd.read() |
|
160 | 160 | finally: |
|
161 | 161 | if fd: |
|
162 | 162 | fd.close() |
|
163 | 163 | return context.memfilectx(f, data, 'l' in fctx.flags(), |
|
164 | 164 | 'x' in fctx.flags(), renamed) |
|
165 | 165 | else: |
|
166 | 166 | return _getnormalcontext(repo.ui, ctx, f, revmap) |
|
167 | 167 | |
|
168 | 168 | dstfiles = [] |
|
169 | 169 | for file in files: |
|
170 | 170 | if lfutil.isstandin(file): |
|
171 | 171 | dstfiles.append(lfutil.splitstandin(file)) |
|
172 | 172 | else: |
|
173 | 173 | dstfiles.append(file) |
|
174 | 174 | # Commit |
|
175 | 175 | _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap) |
|
176 | 176 | |
|
177 | 177 | def _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, lfiles, normalfiles, |
|
178 | 178 | matcher, size, lfiletohash): |
|
179 | 179 | # Convert src parents to dst parents |
|
180 | 180 | parents = _convertparents(ctx, revmap) |
|
181 | 181 | |
|
182 | 182 | # Generate list of changed files |
|
183 | 183 | files = _getchangedfiles(ctx, parents) |
|
184 | 184 | |
|
185 | 185 | dstfiles = [] |
|
186 | 186 | for f in files: |
|
187 | 187 | if f not in lfiles and f not in normalfiles: |
|
188 | 188 | islfile = _islfile(f, ctx, matcher, size) |
|
189 | 189 | # If this file was renamed or copied then copy |
|
190 | 190 | # the largefile-ness of its predecessor |
|
191 | 191 | if f in ctx.manifest(): |
|
192 | 192 | fctx = ctx.filectx(f) |
|
193 | 193 | renamed = fctx.renamed() |
|
194 | 194 | renamedlfile = renamed and renamed[0] in lfiles |
|
195 | 195 | islfile |= renamedlfile |
|
196 | 196 | if 'l' in fctx.flags(): |
|
197 | 197 | if renamedlfile: |
|
198 | 198 | raise util.Abort( |
|
199 | 199 | _('renamed/copied largefile %s becomes symlink') |
|
200 | 200 | % f) |
|
201 | 201 | islfile = False |
|
202 | 202 | if islfile: |
|
203 | 203 | lfiles.add(f) |
|
204 | 204 | else: |
|
205 | 205 | normalfiles.add(f) |
|
206 | 206 | |
|
207 | 207 | if f in lfiles: |
|
208 | 208 | dstfiles.append(lfutil.standin(f)) |
|
209 | 209 | # largefile in manifest if it has not been removed/renamed |
|
210 | 210 | if f in ctx.manifest(): |
|
211 | 211 | fctx = ctx.filectx(f) |
|
212 | 212 | if 'l' in fctx.flags(): |
|
213 | 213 | renamed = fctx.renamed() |
|
214 | 214 | if renamed and renamed[0] in lfiles: |
|
215 | 215 | raise util.Abort(_('largefile %s becomes symlink') % f) |
|
216 | 216 | |
|
217 | 217 | # largefile was modified, update standins |
|
218 | 218 | m = util.sha1('') |
|
219 | 219 | m.update(ctx[f].data()) |
|
220 | 220 | hash = m.hexdigest() |
|
221 | 221 | if f not in lfiletohash or lfiletohash[f] != hash: |
|
222 | 222 | rdst.wwrite(f, ctx[f].data(), ctx[f].flags()) |
|
223 | 223 | executable = 'x' in ctx[f].flags() |
|
224 | 224 | lfutil.writestandin(rdst, lfutil.standin(f), hash, |
|
225 | 225 | executable) |
|
226 | 226 | lfiletohash[f] = hash |
|
227 | 227 | else: |
|
228 | 228 | # normal file |
|
229 | 229 | dstfiles.append(f) |
|
230 | 230 | |
|
231 | 231 | def getfilectx(repo, memctx, f): |
|
232 | 232 | if lfutil.isstandin(f): |
|
233 | 233 | # if the file isn't in the manifest then it was removed |
|
234 | 234 | # or renamed, raise IOError to indicate this |
|
235 | 235 | srcfname = lfutil.splitstandin(f) |
|
236 | 236 | try: |
|
237 | 237 | fctx = ctx.filectx(srcfname) |
|
238 | 238 | except error.LookupError: |
|
239 | 239 | raise IOError |
|
240 | 240 | renamed = fctx.renamed() |
|
241 | 241 | if renamed: |
|
242 | 242 | # standin is always a largefile because largefile-ness |
|
243 | 243 | # doesn't change after rename or copy |
|
244 | 244 | renamed = lfutil.standin(renamed[0]) |
|
245 | 245 | |
|
246 | 246 | return context.memfilectx(f, lfiletohash[srcfname] + '\n', 'l' in |
|
247 | 247 | fctx.flags(), 'x' in fctx.flags(), renamed) |
|
248 | 248 | else: |
|
249 | 249 | return _getnormalcontext(repo.ui, ctx, f, revmap) |
|
250 | 250 | |
|
251 | 251 | # Commit |
|
252 | 252 | _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap) |
|
253 | 253 | |
|
254 | 254 | def _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap): |
|
255 | 255 | mctx = context.memctx(rdst, parents, ctx.description(), dstfiles, |
|
256 | 256 | getfilectx, ctx.user(), ctx.date(), ctx.extra()) |
|
257 | 257 | ret = rdst.commitctx(mctx) |
|
258 | 258 | rdst.setparents(ret) |
|
259 | 259 | revmap[ctx.node()] = rdst.changelog.tip() |
|
260 | 260 | |
|
261 | 261 | # Generate list of changed files |
|
262 | 262 | def _getchangedfiles(ctx, parents): |
|
263 | 263 | files = set(ctx.files()) |
|
264 | 264 | if node.nullid not in parents: |
|
265 | 265 | mc = ctx.manifest() |
|
266 | 266 | mp1 = ctx.parents()[0].manifest() |
|
267 | 267 | mp2 = ctx.parents()[1].manifest() |
|
268 | 268 | files |= (set(mp1) | set(mp2)) - set(mc) |
|
269 | 269 | for f in mc: |
|
270 | 270 | if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None): |
|
271 | 271 | files.add(f) |
|
272 | 272 | return files |
|
273 | 273 | |
|
274 | 274 | # Convert src parents to dst parents |
|
275 | 275 | def _convertparents(ctx, revmap): |
|
276 | 276 | parents = [] |
|
277 | 277 | for p in ctx.parents(): |
|
278 | 278 | parents.append(revmap[p.node()]) |
|
279 | 279 | while len(parents) < 2: |
|
280 | 280 | parents.append(node.nullid) |
|
281 | 281 | return parents |
|
282 | 282 | |
|
283 | 283 | # Get memfilectx for a normal file |
|
284 | 284 | def _getnormalcontext(ui, ctx, f, revmap): |
|
285 | 285 | try: |
|
286 | 286 | fctx = ctx.filectx(f) |
|
287 | 287 | except error.LookupError: |
|
288 | 288 | raise IOError |
|
289 | 289 | renamed = fctx.renamed() |
|
290 | 290 | if renamed: |
|
291 | 291 | renamed = renamed[0] |
|
292 | 292 | |
|
293 | 293 | data = fctx.data() |
|
294 | 294 | if f == '.hgtags': |
|
295 | 295 | data = _converttags (ui, revmap, data) |
|
296 | 296 | return context.memfilectx(f, data, 'l' in fctx.flags(), |
|
297 | 297 | 'x' in fctx.flags(), renamed) |
|
298 | 298 | |
|
299 | 299 | # Remap tag data using a revision map |
|
300 | 300 | def _converttags(ui, revmap, data): |
|
301 | 301 | newdata = [] |
|
302 | 302 | for line in data.splitlines(): |
|
303 | 303 | try: |
|
304 | 304 | id, name = line.split(' ', 1) |
|
305 | 305 | except ValueError: |
|
306 | ui.warn(_('skipping incorrectly formatted tag %s\n' | |
|
307 |
% line) |
|
|
306 | ui.warn(_('skipping incorrectly formatted tag %s\n') | |
|
307 | % line) | |
|
308 | 308 | continue |
|
309 | 309 | try: |
|
310 | 310 | newid = node.bin(id) |
|
311 | 311 | except TypeError: |
|
312 | ui.warn(_('skipping incorrectly formatted id %s\n' | |
|
313 |
% id) |
|
|
312 | ui.warn(_('skipping incorrectly formatted id %s\n') | |
|
313 | % id) | |
|
314 | 314 | continue |
|
315 | 315 | try: |
|
316 | 316 | newdata.append('%s %s\n' % (node.hex(revmap[newid]), |
|
317 | 317 | name)) |
|
318 | 318 | except KeyError: |
|
319 | 319 | ui.warn(_('no mapping for id %s\n') % id) |
|
320 | 320 | continue |
|
321 | 321 | return ''.join(newdata) |
|
322 | 322 | |
|
323 | 323 | def _islfile(file, ctx, matcher, size): |
|
324 | 324 | '''Return true if file should be considered a largefile, i.e. |
|
325 | 325 | matcher matches it or it is larger than size.''' |
|
326 | 326 | # never store special .hg* files as largefiles |
|
327 | 327 | if file == '.hgtags' or file == '.hgignore' or file == '.hgsigs': |
|
328 | 328 | return False |
|
329 | 329 | if matcher and matcher(file): |
|
330 | 330 | return True |
|
331 | 331 | try: |
|
332 | 332 | return ctx.filectx(file).size() >= size * 1024 * 1024 |
|
333 | 333 | except error.LookupError: |
|
334 | 334 | return False |
|
335 | 335 | |
|
336 | 336 | def uploadlfiles(ui, rsrc, rdst, files): |
|
337 | 337 | '''upload largefiles to the central store''' |
|
338 | 338 | |
|
339 | 339 | if not files: |
|
340 | 340 | return |
|
341 | 341 | |
|
342 | 342 | store = basestore._openstore(rsrc, rdst, put=True) |
|
343 | 343 | |
|
344 | 344 | at = 0 |
|
345 | 345 | ui.debug("sending statlfile command for %d largefiles\n" % len(files)) |
|
346 | 346 | retval = store.exists(files) |
|
347 | 347 | files = filter(lambda h: not retval[h], files) |
|
348 | 348 | ui.debug("%d largefiles need to be uploaded\n" % len(files)) |
|
349 | 349 | |
|
350 | 350 | for hash in files: |
|
351 | 351 | ui.progress(_('uploading largefiles'), at, unit='largefile', |
|
352 | 352 | total=len(files)) |
|
353 | 353 | source = lfutil.findfile(rsrc, hash) |
|
354 | 354 | if not source: |
|
355 | 355 | raise util.Abort(_('largefile %s missing from store' |
|
356 | 356 | ' (needs to be uploaded)') % hash) |
|
357 | 357 | # XXX check for errors here |
|
358 | 358 | store.put(source, hash) |
|
359 | 359 | at += 1 |
|
360 | 360 | ui.progress(_('uploading largefiles'), None) |
|
361 | 361 | |
|
362 | 362 | def verifylfiles(ui, repo, all=False, contents=False): |
|
363 | 363 | '''Verify that every largefile revision in the current changeset |
|
364 | 364 | exists in the central store. With --contents, also verify that |
|
365 | 365 | the contents of each local largefile file revision are correct (SHA-1 hash |
|
366 | 366 | matches the revision ID). With --all, check every changeset in |
|
367 | 367 | this repository.''' |
|
368 | 368 | if all: |
|
369 | 369 | # Pass a list to the function rather than an iterator because we know a |
|
370 | 370 | # list will work. |
|
371 | 371 | revs = range(len(repo)) |
|
372 | 372 | else: |
|
373 | 373 | revs = ['.'] |
|
374 | 374 | |
|
375 | 375 | store = basestore._openstore(repo) |
|
376 | 376 | return store.verify(revs, contents=contents) |
|
377 | 377 | |
|
378 | 378 | def debugdirstate(ui, repo): |
|
379 | 379 | '''Show basic information for the largefiles dirstate''' |
|
380 | 380 | lfdirstate = lfutil.openlfdirstate(ui, repo) |
|
381 | 381 | for file_, ent in sorted(lfdirstate._map.iteritems()): |
|
382 | 382 | mode = '%3o' % (ent[1] & 0777 & ~util.umask) |
|
383 | 383 | ui.write("%c %s %10d %s\n" % (ent[0], mode, ent[2], file_)) |
|
384 | 384 | |
|
385 | 385 | def cachelfiles(ui, repo, node, filelist=None): |
|
386 | 386 | '''cachelfiles ensures that all largefiles needed by the specified revision |
|
387 | 387 | are present in the repository's largefile cache. |
|
388 | 388 | |
|
389 | 389 | returns a tuple (cached, missing). cached is the list of files downloaded |
|
390 | 390 | by this operation; missing is the list of files that were needed but could |
|
391 | 391 | not be found.''' |
|
392 | 392 | lfiles = lfutil.listlfiles(repo, node) |
|
393 | 393 | if filelist: |
|
394 | 394 | lfiles = set(lfiles) & set(filelist) |
|
395 | 395 | toget = [] |
|
396 | 396 | |
|
397 | 397 | for lfile in lfiles: |
|
398 | 398 | try: |
|
399 | 399 | expectedhash = repo[node][lfutil.standin(lfile)].data().strip() |
|
400 | 400 | except IOError, err: |
|
401 | 401 | if err.errno == errno.ENOENT: |
|
402 | 402 | continue # node must be None and standin wasn't found in wctx |
|
403 | 403 | raise |
|
404 | 404 | if not lfutil.findfile(repo, expectedhash): |
|
405 | 405 | toget.append((lfile, expectedhash)) |
|
406 | 406 | |
|
407 | 407 | if toget: |
|
408 | 408 | store = basestore._openstore(repo) |
|
409 | 409 | ret = store.get(toget) |
|
410 | 410 | return ret |
|
411 | 411 | |
|
412 | 412 | return ([], []) |
|
413 | 413 | |
|
414 | 414 | def downloadlfiles(ui, repo, rev=None): |
|
415 | 415 | matchfn = scmutil.match(repo[None], |
|
416 | 416 | [repo.wjoin(lfutil.shortname)], {}) |
|
417 | 417 | def prepare(ctx, fns): |
|
418 | 418 | pass |
|
419 | 419 | totalsuccess = 0 |
|
420 | 420 | totalmissing = 0 |
|
421 | 421 | if rev != []: # walkchangerevs on empty list would return all revs |
|
422 | 422 | for ctx in cmdutil.walkchangerevs(repo, matchfn, {'rev' : rev}, |
|
423 | 423 | prepare): |
|
424 | 424 | success, missing = cachelfiles(ui, repo, ctx.node()) |
|
425 | 425 | totalsuccess += len(success) |
|
426 | 426 | totalmissing += len(missing) |
|
427 | 427 | ui.status(_("%d additional largefiles cached\n") % totalsuccess) |
|
428 | 428 | if totalmissing > 0: |
|
429 | 429 | ui.status(_("%d largefiles failed to download\n") % totalmissing) |
|
430 | 430 | return totalsuccess, totalmissing |
|
431 | 431 | |
|
432 | 432 | def updatelfiles(ui, repo, filelist=None, printmessage=True): |
|
433 | 433 | wlock = repo.wlock() |
|
434 | 434 | try: |
|
435 | 435 | lfdirstate = lfutil.openlfdirstate(ui, repo) |
|
436 | 436 | lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) |
|
437 | 437 | |
|
438 | 438 | if filelist is not None: |
|
439 | 439 | lfiles = [f for f in lfiles if f in filelist] |
|
440 | 440 | |
|
441 | 441 | update = {} |
|
442 | 442 | updated, removed = 0, 0 |
|
443 | 443 | for lfile in lfiles: |
|
444 | 444 | abslfile = repo.wjoin(lfile) |
|
445 | 445 | absstandin = repo.wjoin(lfutil.standin(lfile)) |
|
446 | 446 | if os.path.exists(absstandin): |
|
447 | 447 | if (os.path.exists(absstandin + '.orig') and |
|
448 | 448 | os.path.exists(abslfile)): |
|
449 | 449 | shutil.copyfile(abslfile, abslfile + '.orig') |
|
450 | 450 | expecthash = lfutil.readstandin(repo, lfile) |
|
451 | 451 | if (expecthash != '' and |
|
452 | 452 | (not os.path.exists(abslfile) or |
|
453 | 453 | expecthash != lfutil.hashfile(abslfile))): |
|
454 | 454 | if lfile not in repo[None]: # not switched to normal file |
|
455 | 455 | util.unlinkpath(abslfile, ignoremissing=True) |
|
456 | 456 | # use normallookup() to allocate entry in largefiles |
|
457 | 457 | # dirstate, because lack of it misleads |
|
458 | 458 | # lfilesrepo.status() into recognition that such cache |
|
459 | 459 | # missing files are REMOVED. |
|
460 | 460 | lfdirstate.normallookup(lfile) |
|
461 | 461 | update[lfile] = expecthash |
|
462 | 462 | else: |
|
463 | 463 | # Remove lfiles for which the standin is deleted, unless the |
|
464 | 464 | # lfile is added to the repository again. This happens when a |
|
465 | 465 | # largefile is converted back to a normal file: the standin |
|
466 | 466 | # disappears, but a new (normal) file appears as the lfile. |
|
467 | 467 | if (os.path.exists(abslfile) and |
|
468 | 468 | repo.dirstate.normalize(lfile) not in repo[None]): |
|
469 | 469 | util.unlinkpath(abslfile) |
|
470 | 470 | removed += 1 |
|
471 | 471 | |
|
472 | 472 | # largefile processing might be slow and be interrupted - be prepared |
|
473 | 473 | lfdirstate.write() |
|
474 | 474 | |
|
475 | 475 | if lfiles: |
|
476 | 476 | if printmessage: |
|
477 | 477 | ui.status(_('getting changed largefiles\n')) |
|
478 | 478 | cachelfiles(ui, repo, None, lfiles) |
|
479 | 479 | |
|
480 | 480 | for lfile in lfiles: |
|
481 | 481 | update1 = 0 |
|
482 | 482 | |
|
483 | 483 | expecthash = update.get(lfile) |
|
484 | 484 | if expecthash: |
|
485 | 485 | if not lfutil.copyfromcache(repo, expecthash, lfile): |
|
486 | 486 | # failed ... but already removed and set to normallookup |
|
487 | 487 | continue |
|
488 | 488 | # Synchronize largefile dirstate to the last modified |
|
489 | 489 | # time of the file |
|
490 | 490 | lfdirstate.normal(lfile) |
|
491 | 491 | update1 = 1 |
|
492 | 492 | |
|
493 | 493 | # copy the state of largefile standin from the repository's |
|
494 | 494 | # dirstate to its state in the lfdirstate. |
|
495 | 495 | abslfile = repo.wjoin(lfile) |
|
496 | 496 | absstandin = repo.wjoin(lfutil.standin(lfile)) |
|
497 | 497 | if os.path.exists(absstandin): |
|
498 | 498 | mode = os.stat(absstandin).st_mode |
|
499 | 499 | if mode != os.stat(abslfile).st_mode: |
|
500 | 500 | os.chmod(abslfile, mode) |
|
501 | 501 | update1 = 1 |
|
502 | 502 | |
|
503 | 503 | updated += update1 |
|
504 | 504 | |
|
505 | 505 | state = repo.dirstate[lfutil.standin(lfile)] |
|
506 | 506 | if state == 'n': |
|
507 | 507 | # When rebasing, we need to synchronize the standin and the |
|
508 | 508 | # largefile, because otherwise the largefile will get reverted. |
|
509 | 509 | # But for commit's sake, we have to mark the file as unclean. |
|
510 | 510 | if getattr(repo, "_isrebasing", False): |
|
511 | 511 | lfdirstate.normallookup(lfile) |
|
512 | 512 | else: |
|
513 | 513 | lfdirstate.normal(lfile) |
|
514 | 514 | elif state == 'r': |
|
515 | 515 | lfdirstate.remove(lfile) |
|
516 | 516 | elif state == 'a': |
|
517 | 517 | lfdirstate.add(lfile) |
|
518 | 518 | elif state == '?': |
|
519 | 519 | lfdirstate.drop(lfile) |
|
520 | 520 | |
|
521 | 521 | lfdirstate.write() |
|
522 | 522 | if printmessage and lfiles: |
|
523 | 523 | ui.status(_('%d largefiles updated, %d removed\n') % (updated, |
|
524 | 524 | removed)) |
|
525 | 525 | finally: |
|
526 | 526 | wlock.release() |
|
527 | 527 | |
|
528 | 528 | def lfpull(ui, repo, source="default", **opts): |
|
529 | 529 | """pull largefiles for the specified revisions from the specified source |
|
530 | 530 | |
|
531 | 531 | Pull largefiles that are referenced from local changesets but missing |
|
532 | 532 | locally, pulling from a remote repository to the local cache. |
|
533 | 533 | |
|
534 | 534 | If SOURCE is omitted, the 'default' path will be used. |
|
535 | 535 | See :hg:`help urls` for more information. |
|
536 | 536 | |
|
537 | 537 | .. container:: verbose |
|
538 | 538 | |
|
539 | 539 | Some examples: |
|
540 | 540 | |
|
541 | 541 | - pull largefiles for all branch heads:: |
|
542 | 542 | |
|
543 | 543 | hg lfpull -r "head() and not closed()" |
|
544 | 544 | |
|
545 | 545 | - pull largefiles on the default branch:: |
|
546 | 546 | |
|
547 | 547 | hg lfpull -r "branch(default)" |
|
548 | 548 | """ |
|
549 | 549 | repo.lfpullsource = source |
|
550 | 550 | |
|
551 | 551 | revs = opts.get('rev', []) |
|
552 | 552 | if not revs: |
|
553 | 553 | raise util.Abort(_('no revisions specified')) |
|
554 | 554 | revs = scmutil.revrange(repo, revs) |
|
555 | 555 | |
|
556 | 556 | numcached = 0 |
|
557 | 557 | for rev in revs: |
|
558 | 558 | ui.note(_('pulling largefiles for revision %s\n') % rev) |
|
559 | 559 | (cached, missing) = cachelfiles(ui, repo, rev) |
|
560 | 560 | numcached += len(cached) |
|
561 | 561 | ui.status(_("%d largefiles cached\n") % numcached) |
|
562 | 562 | |
|
563 | 563 | # -- hg commands declarations ------------------------------------------------ |
|
564 | 564 | |
|
565 | 565 | cmdtable = { |
|
566 | 566 | 'lfconvert': (lfconvert, |
|
567 | 567 | [('s', 'size', '', |
|
568 | 568 | _('minimum size (MB) for files to be converted ' |
|
569 | 569 | 'as largefiles'), |
|
570 | 570 | 'SIZE'), |
|
571 | 571 | ('', 'to-normal', False, |
|
572 | 572 | _('convert from a largefiles repo to a normal repo')), |
|
573 | 573 | ], |
|
574 | 574 | _('hg lfconvert SOURCE DEST [FILE ...]')), |
|
575 | 575 | 'lfpull': (lfpull, |
|
576 | 576 | [('r', 'rev', [], _('pull largefiles for these revisions')) |
|
577 | 577 | ] + commands.remoteopts, |
|
578 | 578 | _('-r REV... [-e CMD] [--remotecmd CMD] [SOURCE]') |
|
579 | 579 | ), |
|
580 | 580 | } |
|
581 | 581 | |
|
582 | 582 | commands.inferrepo += " lfconvert" |
|
1 | NO CONTENT: modified file | |
The requested commit or file is too big and content was truncated. Show full diff |
|
1 | NO CONTENT: modified file | |
The requested commit or file is too big and content was truncated. Show full diff |
|
1 | NO CONTENT: modified file | |
The requested commit or file is too big and content was truncated. Show full diff |
|
1 | NO CONTENT: modified file | |
The requested commit or file is too big and content was truncated. Show full diff |
|
1 | NO CONTENT: modified file | |
The requested commit or file is too big and content was truncated. Show full diff |
|
1 | NO CONTENT: modified file | |
The requested commit or file is too big and content was truncated. Show full diff |
|
1 | NO CONTENT: modified file | |
The requested commit or file is too big and content was truncated. Show full diff |
General Comments 0
You need to be logged in to leave comments.
Login now