##// END OF EJS Templates
dirstate: deal with read-race for python code using rust object...
marmoute -
r51133:c9066fc6 stable
parent child Browse files
Show More
@@ -1,733 +1,732
1 1 # dirstatemap.py
2 2 #
3 3 # This software may be used and distributed according to the terms of the
4 4 # GNU General Public License version 2 or any later version.
5 5
6 6
7 7 from .i18n import _
8 8
9 9 from . import (
10 10 error,
11 11 pathutil,
12 12 policy,
13 13 testing,
14 14 txnutil,
15 15 util,
16 16 )
17 17
18 18 from .dirstateutils import (
19 19 docket as docketmod,
20 20 v2,
21 21 )
22 22
23 23 parsers = policy.importmod('parsers')
24 24 rustmod = policy.importrust('dirstate')
25 25
26 26 propertycache = util.propertycache
27 27
28 28 if rustmod is None:
29 29 DirstateItem = parsers.DirstateItem
30 30 else:
31 31 DirstateItem = rustmod.DirstateItem
32 32
33 33 rangemask = 0x7FFFFFFF
34 34
35 35 WRITE_MODE_AUTO = 0
36 36 WRITE_MODE_FORCE_NEW = 1
37 37 WRITE_MODE_FORCE_APPEND = 2
38 38
39 39
40 40 V2_MAX_READ_ATTEMPTS = 5
41 41
42 42
43 43 class _dirstatemapcommon:
44 44 """
45 45 Methods that are identical for both implementations of the dirstatemap
46 46 class, with and without Rust extensions enabled.
47 47 """
48 48
49 49 # please pytype
50 50
51 51 _map = None
52 52 copymap = None
53 53
54 54 def __init__(self, ui, opener, root, nodeconstants, use_dirstate_v2):
55 55 self._use_dirstate_v2 = use_dirstate_v2
56 56 self._nodeconstants = nodeconstants
57 57 self._ui = ui
58 58 self._opener = opener
59 59 self._root = root
60 60 self._filename = b'dirstate'
61 61 self._nodelen = 20 # Also update Rust code when changing this!
62 62 self._parents = None
63 63 self._dirtyparents = False
64 64 self._docket = None
65 65 write_mode = ui.config(b"devel", b"dirstate.v2.data_update_mode")
66 66 if write_mode == b"auto":
67 67 self._write_mode = WRITE_MODE_AUTO
68 68 elif write_mode == b"force-append":
69 69 self._write_mode = WRITE_MODE_FORCE_APPEND
70 70 elif write_mode == b"force-new":
71 71 self._write_mode = WRITE_MODE_FORCE_NEW
72 72 else:
73 73 # unknown value, fallback to default
74 74 self._write_mode = WRITE_MODE_AUTO
75 75
76 76 # for consistent view between _pl() and _read() invocations
77 77 self._pendingmode = None
78 78
79 79 def preload(self):
80 80 """Loads the underlying data, if it's not already loaded"""
81 81 self._map
82 82
83 83 def get(self, key, default=None):
84 84 return self._map.get(key, default)
85 85
86 86 def __len__(self):
87 87 return len(self._map)
88 88
89 89 def __iter__(self):
90 90 return iter(self._map)
91 91
92 92 def __contains__(self, key):
93 93 return key in self._map
94 94
95 95 def __getitem__(self, item):
96 96 return self._map[item]
97 97
98 98 ### disk interaction
99 99
100 100 def _opendirstatefile(self):
101 101 fp, mode = txnutil.trypending(self._root, self._opener, self._filename)
102 102 if self._pendingmode is not None and self._pendingmode != mode:
103 103 fp.close()
104 104 raise error.Abort(
105 105 _(b'working directory state may be changed parallelly')
106 106 )
107 107 self._pendingmode = mode
108 108 return fp
109 109
110 110 def _readdirstatefile(self, size=-1):
111 111 try:
112 112 with self._opendirstatefile() as fp:
113 113 return fp.read(size)
114 114 except FileNotFoundError:
115 115 # File doesn't exist, so the current state is empty
116 116 return b''
117 117
118 118 @property
119 119 def docket(self):
120 120 if not self._docket:
121 121 if not self._use_dirstate_v2:
122 122 raise error.ProgrammingError(
123 123 b'dirstate only has a docket in v2 format'
124 124 )
125 125 self._docket = docketmod.DirstateDocket.parse(
126 126 self._readdirstatefile(), self._nodeconstants
127 127 )
128 128 return self._docket
129 129
130 130 def _read_v2_data(self):
131 131 data = None
132 132 attempts = 0
133 133 while attempts < V2_MAX_READ_ATTEMPTS:
134 134 attempts += 1
135 135 try:
136 # TODO: use mmap when possible
136 137 data = self._opener.read(self.docket.data_filename())
137 138 except FileNotFoundError:
138 139 # read race detected between docket and data file
139 140 # reload the docket and retry
140 141 self._docket = None
141 142 if data is None:
142 143 assert attempts >= V2_MAX_READ_ATTEMPTS
143 144 msg = b"dirstate read race happened %d times in a row"
144 145 msg %= attempts
145 146 raise error.Abort(msg)
146 147 return self._opener.read(self.docket.data_filename())
147 148
148 149 def write_v2_no_append(self, tr, st, meta, packed):
149 150 old_docket = self.docket
150 151 new_docket = docketmod.DirstateDocket.with_new_uuid(
151 152 self.parents(), len(packed), meta
152 153 )
153 154 if old_docket.uuid == new_docket.uuid:
154 155 raise error.ProgrammingError(b'dirstate docket name collision')
155 156 data_filename = new_docket.data_filename()
156 157 self._opener.write(data_filename, packed)
157 158 # Write the new docket after the new data file has been
158 159 # written. Because `st` was opened with `atomictemp=True`,
159 160 # the actual `.hg/dirstate` file is only affected on close.
160 161 st.write(new_docket.serialize())
161 162 st.close()
162 163 # Remove the old data file after the new docket pointing to
163 164 # the new data file was written.
164 165 if old_docket.uuid:
165 166 data_filename = old_docket.data_filename()
166 167 unlink = lambda _tr=None: self._opener.unlink(data_filename)
167 168 if tr:
168 169 category = b"dirstate-v2-clean-" + old_docket.uuid
169 170 tr.addpostclose(category, unlink)
170 171 else:
171 172 unlink()
172 173 self._docket = new_docket
173 174
174 175 ### reading/setting parents
175 176
176 177 def parents(self):
177 178 if not self._parents:
178 179 if self._use_dirstate_v2:
179 180 self._parents = self.docket.parents
180 181 else:
181 182 read_len = self._nodelen * 2
182 183 st = self._readdirstatefile(read_len)
183 184 l = len(st)
184 185 if l == read_len:
185 186 self._parents = (
186 187 st[: self._nodelen],
187 188 st[self._nodelen : 2 * self._nodelen],
188 189 )
189 190 elif l == 0:
190 191 self._parents = (
191 192 self._nodeconstants.nullid,
192 193 self._nodeconstants.nullid,
193 194 )
194 195 else:
195 196 raise error.Abort(
196 197 _(b'working directory state appears damaged!')
197 198 )
198 199
199 200 return self._parents
200 201
201 202
202 203 class dirstatemap(_dirstatemapcommon):
203 204 """Map encapsulating the dirstate's contents.
204 205
205 206 The dirstate contains the following state:
206 207
207 208 - `identity` is the identity of the dirstate file, which can be used to
208 209 detect when changes have occurred to the dirstate file.
209 210
210 211 - `parents` is a pair containing the parents of the working copy. The
211 212 parents are updated by calling `setparents`.
212 213
213 214 - the state map maps filenames to tuples of (state, mode, size, mtime),
214 215 where state is a single character representing 'normal', 'added',
215 216 'removed', or 'merged'. It is read by treating the dirstate as a
216 217 dict. File state is updated by calling various methods (see each
217 218 documentation for details):
218 219
219 220 - `reset_state`,
220 221 - `set_tracked`
221 222 - `set_untracked`
222 223 - `set_clean`
223 224 - `set_possibly_dirty`
224 225
225 226 - `copymap` maps destination filenames to their source filename.
226 227
227 228 The dirstate also provides the following views onto the state:
228 229
229 230 - `filefoldmap` is a dict mapping normalized filenames to the denormalized
230 231 form that they appear as in the dirstate.
231 232
232 233 - `dirfoldmap` is a dict mapping normalized directory names to the
233 234 denormalized form that they appear as in the dirstate.
234 235 """
235 236
236 237 ### Core data storage and access
237 238
238 239 @propertycache
239 240 def _map(self):
240 241 self._map = {}
241 242 self.read()
242 243 return self._map
243 244
244 245 @propertycache
245 246 def copymap(self):
246 247 self.copymap = {}
247 248 self._map
248 249 return self.copymap
249 250
250 251 def clear(self):
251 252 self._map.clear()
252 253 self.copymap.clear()
253 254 self.setparents(self._nodeconstants.nullid, self._nodeconstants.nullid)
254 255 util.clearcachedproperty(self, b"_dirs")
255 256 util.clearcachedproperty(self, b"_alldirs")
256 257 util.clearcachedproperty(self, b"filefoldmap")
257 258 util.clearcachedproperty(self, b"dirfoldmap")
258 259
259 260 def items(self):
260 261 return self._map.items()
261 262
262 263 # forward for python2,3 compat
263 264 iteritems = items
264 265
265 266 def debug_iter(self, all):
266 267 """
267 268 Return an iterator of (filename, state, mode, size, mtime) tuples
268 269
269 270 `all` is unused when Rust is not enabled
270 271 """
271 272 for (filename, item) in self.items():
272 273 yield (filename, item.state, item.mode, item.size, item.mtime)
273 274
274 275 def keys(self):
275 276 return self._map.keys()
276 277
277 278 ### reading/setting parents
278 279
279 280 def setparents(self, p1, p2, fold_p2=False):
280 281 self._parents = (p1, p2)
281 282 self._dirtyparents = True
282 283 copies = {}
283 284 if fold_p2:
284 285 for f, s in self._map.items():
285 286 # Discard "merged" markers when moving away from a merge state
286 287 if s.p2_info:
287 288 source = self.copymap.pop(f, None)
288 289 if source:
289 290 copies[f] = source
290 291 s.drop_merge_data()
291 292 return copies
292 293
293 294 ### disk interaction
294 295
295 296 def read(self):
296 297 # ignore HG_PENDING because identity is used only for writing
297 298 self.identity = util.filestat.frompath(
298 299 self._opener.join(self._filename)
299 300 )
300 301
301 302 testing.wait_on_cfg(self._ui, b'dirstate.pre-read-file')
302 303 if self._use_dirstate_v2:
303 304
304 305 if not self.docket.uuid:
305 306 return
306 307 testing.wait_on_cfg(self._ui, b'dirstate.post-docket-read-file')
307 308 st = self._read_v2_data()
308 309 else:
309 310 st = self._readdirstatefile()
310 311
311 312 if not st:
312 313 return
313 314
314 315 # TODO: adjust this estimate for dirstate-v2
315 316 if util.safehasattr(parsers, b'dict_new_presized'):
316 317 # Make an estimate of the number of files in the dirstate based on
317 318 # its size. This trades wasting some memory for avoiding costly
318 319 # resizes. Each entry have a prefix of 17 bytes followed by one or
319 320 # two path names. Studies on various large-scale real-world repositories
320 321 # found 54 bytes a reasonable upper limit for the average path names.
321 322 # Copy entries are ignored for the sake of this estimate.
322 323 self._map = parsers.dict_new_presized(len(st) // 71)
323 324
324 325 # Python's garbage collector triggers a GC each time a certain number
325 326 # of container objects (the number being defined by
326 327 # gc.get_threshold()) are allocated. parse_dirstate creates a tuple
327 328 # for each file in the dirstate. The C version then immediately marks
328 329 # them as not to be tracked by the collector. However, this has no
329 330 # effect on when GCs are triggered, only on what objects the GC looks
330 331 # into. This means that O(number of files) GCs are unavoidable.
331 332 # Depending on when in the process's lifetime the dirstate is parsed,
332 333 # this can get very expensive. As a workaround, disable GC while
333 334 # parsing the dirstate.
334 335 #
335 336 # (we cannot decorate the function directly since it is in a C module)
336 337 if self._use_dirstate_v2:
337 338 p = self.docket.parents
338 339 meta = self.docket.tree_metadata
339 340 parse_dirstate = util.nogc(v2.parse_dirstate)
340 341 parse_dirstate(self._map, self.copymap, st, meta)
341 342 else:
342 343 parse_dirstate = util.nogc(parsers.parse_dirstate)
343 344 p = parse_dirstate(self._map, self.copymap, st)
344 345 if not self._dirtyparents:
345 346 self.setparents(*p)
346 347
347 348 # Avoid excess attribute lookups by fast pathing certain checks
348 349 self.__contains__ = self._map.__contains__
349 350 self.__getitem__ = self._map.__getitem__
350 351 self.get = self._map.get
351 352
352 353 def write(self, tr, st):
353 354 if self._use_dirstate_v2:
354 355 packed, meta = v2.pack_dirstate(self._map, self.copymap)
355 356 self.write_v2_no_append(tr, st, meta, packed)
356 357 else:
357 358 packed = parsers.pack_dirstate(
358 359 self._map, self.copymap, self.parents()
359 360 )
360 361 st.write(packed)
361 362 st.close()
362 363 self._dirtyparents = False
363 364
364 365 @propertycache
365 366 def identity(self):
366 367 self._map
367 368 return self.identity
368 369
369 370 ### code related to maintaining and accessing "extra" property
370 371 # (e.g. "has_dir")
371 372
372 373 def _dirs_incr(self, filename, old_entry=None):
373 374 """increment the dirstate counter if applicable"""
374 375 if (
375 376 old_entry is None or old_entry.removed
376 377 ) and "_dirs" in self.__dict__:
377 378 self._dirs.addpath(filename)
378 379 if old_entry is None and "_alldirs" in self.__dict__:
379 380 self._alldirs.addpath(filename)
380 381
381 382 def _dirs_decr(self, filename, old_entry=None, remove_variant=False):
382 383 """decrement the dirstate counter if applicable"""
383 384 if old_entry is not None:
384 385 if "_dirs" in self.__dict__ and not old_entry.removed:
385 386 self._dirs.delpath(filename)
386 387 if "_alldirs" in self.__dict__ and not remove_variant:
387 388 self._alldirs.delpath(filename)
388 389 elif remove_variant and "_alldirs" in self.__dict__:
389 390 self._alldirs.addpath(filename)
390 391 if "filefoldmap" in self.__dict__:
391 392 normed = util.normcase(filename)
392 393 self.filefoldmap.pop(normed, None)
393 394
394 395 @propertycache
395 396 def filefoldmap(self):
396 397 """Returns a dictionary mapping normalized case paths to their
397 398 non-normalized versions.
398 399 """
399 400 try:
400 401 makefilefoldmap = parsers.make_file_foldmap
401 402 except AttributeError:
402 403 pass
403 404 else:
404 405 return makefilefoldmap(
405 406 self._map, util.normcasespec, util.normcasefallback
406 407 )
407 408
408 409 f = {}
409 410 normcase = util.normcase
410 411 for name, s in self._map.items():
411 412 if not s.removed:
412 413 f[normcase(name)] = name
413 414 f[b'.'] = b'.' # prevents useless util.fspath() invocation
414 415 return f
415 416
416 417 @propertycache
417 418 def dirfoldmap(self):
418 419 f = {}
419 420 normcase = util.normcase
420 421 for name in self._dirs:
421 422 f[normcase(name)] = name
422 423 return f
423 424
424 425 def hastrackeddir(self, d):
425 426 """
426 427 Returns True if the dirstate contains a tracked (not removed) file
427 428 in this directory.
428 429 """
429 430 return d in self._dirs
430 431
431 432 def hasdir(self, d):
432 433 """
433 434 Returns True if the dirstate contains a file (tracked or removed)
434 435 in this directory.
435 436 """
436 437 return d in self._alldirs
437 438
438 439 @propertycache
439 440 def _dirs(self):
440 441 return pathutil.dirs(self._map, only_tracked=True)
441 442
442 443 @propertycache
443 444 def _alldirs(self):
444 445 return pathutil.dirs(self._map)
445 446
446 447 ### code related to manipulation of entries and copy-sources
447 448
448 449 def reset_state(
449 450 self,
450 451 filename,
451 452 wc_tracked=False,
452 453 p1_tracked=False,
453 454 p2_info=False,
454 455 has_meaningful_mtime=True,
455 456 parentfiledata=None,
456 457 ):
457 458 """Set a entry to a given state, diregarding all previous state
458 459
459 460 This is to be used by the part of the dirstate API dedicated to
460 461 adjusting the dirstate after a update/merge.
461 462
462 463 note: calling this might result to no entry existing at all if the
463 464 dirstate map does not see any point at having one for this file
464 465 anymore.
465 466 """
466 467 # copy information are now outdated
467 468 # (maybe new information should be in directly passed to this function)
468 469 self.copymap.pop(filename, None)
469 470
470 471 if not (p1_tracked or p2_info or wc_tracked):
471 472 old_entry = self._map.get(filename)
472 473 self._drop_entry(filename)
473 474 self._dirs_decr(filename, old_entry=old_entry)
474 475 return
475 476
476 477 old_entry = self._map.get(filename)
477 478 self._dirs_incr(filename, old_entry)
478 479 entry = DirstateItem(
479 480 wc_tracked=wc_tracked,
480 481 p1_tracked=p1_tracked,
481 482 p2_info=p2_info,
482 483 has_meaningful_mtime=has_meaningful_mtime,
483 484 parentfiledata=parentfiledata,
484 485 )
485 486 self._map[filename] = entry
486 487
487 488 def set_tracked(self, filename):
488 489 new = False
489 490 entry = self.get(filename)
490 491 if entry is None:
491 492 self._dirs_incr(filename)
492 493 entry = DirstateItem(
493 494 wc_tracked=True,
494 495 )
495 496
496 497 self._map[filename] = entry
497 498 new = True
498 499 elif not entry.tracked:
499 500 self._dirs_incr(filename, entry)
500 501 entry.set_tracked()
501 502 self._refresh_entry(filename, entry)
502 503 new = True
503 504 else:
504 505 # XXX This is probably overkill for more case, but we need this to
505 506 # fully replace the `normallookup` call with `set_tracked` one.
506 507 # Consider smoothing this in the future.
507 508 entry.set_possibly_dirty()
508 509 self._refresh_entry(filename, entry)
509 510 return new
510 511
511 512 def set_untracked(self, f):
512 513 """Mark a file as no longer tracked in the dirstate map"""
513 514 entry = self.get(f)
514 515 if entry is None:
515 516 return False
516 517 else:
517 518 self._dirs_decr(f, old_entry=entry, remove_variant=not entry.added)
518 519 if not entry.p2_info:
519 520 self.copymap.pop(f, None)
520 521 entry.set_untracked()
521 522 self._refresh_entry(f, entry)
522 523 return True
523 524
524 525 def set_clean(self, filename, mode, size, mtime):
525 526 """mark a file as back to a clean state"""
526 527 entry = self[filename]
527 528 size = size & rangemask
528 529 entry.set_clean(mode, size, mtime)
529 530 self._refresh_entry(filename, entry)
530 531 self.copymap.pop(filename, None)
531 532
532 533 def set_possibly_dirty(self, filename):
533 534 """record that the current state of the file on disk is unknown"""
534 535 entry = self[filename]
535 536 entry.set_possibly_dirty()
536 537 self._refresh_entry(filename, entry)
537 538
538 539 def _refresh_entry(self, f, entry):
539 540 """record updated state of an entry"""
540 541 if not entry.any_tracked:
541 542 self._map.pop(f, None)
542 543
543 544 def _drop_entry(self, f):
544 545 """remove any entry for file f
545 546
546 547 This should also drop associated copy information
547 548
548 549 The fact we actually need to drop it is the responsability of the caller"""
549 550 self._map.pop(f, None)
550 551 self.copymap.pop(f, None)
551 552
552 553
553 554 if rustmod is not None:
554 555
555 556 class dirstatemap(_dirstatemapcommon):
556 557
557 558 ### Core data storage and access
558 559
559 560 @propertycache
560 561 def _map(self):
561 562 """
562 563 Fills the Dirstatemap when called.
563 564 """
564 565 # ignore HG_PENDING because identity is used only for writing
565 566 self.identity = util.filestat.frompath(
566 567 self._opener.join(self._filename)
567 568 )
568 569
569 570 testing.wait_on_cfg(self._ui, b'dirstate.pre-read-file')
570 571 if self._use_dirstate_v2:
571 if self.docket.uuid:
572 testing.wait_on_cfg(
573 self._ui, b'dirstate.post-docket-read-file'
574 )
575 # TODO: use mmap when possible
576 data = self._opener.read(self.docket.data_filename())
572 self.docket # load the data if needed
573 testing.wait_on_cfg(self._ui, b'dirstate.post-docket-read-file')
574 if not self.docket.uuid:
575 data = b''
577 576 else:
578 data = b''
577 data = self._read_v2_data()
579 578 self._map = rustmod.DirstateMap.new_v2(
580 579 data, self.docket.data_size, self.docket.tree_metadata
581 580 )
582 581 parents = self.docket.parents
583 582 else:
584 583 self._map, parents = rustmod.DirstateMap.new_v1(
585 584 self._readdirstatefile()
586 585 )
587 586
588 587 if parents and not self._dirtyparents:
589 588 self.setparents(*parents)
590 589
591 590 self.__contains__ = self._map.__contains__
592 591 self.__getitem__ = self._map.__getitem__
593 592 self.get = self._map.get
594 593 return self._map
595 594
596 595 @property
597 596 def copymap(self):
598 597 return self._map.copymap()
599 598
600 599 def debug_iter(self, all):
601 600 """
602 601 Return an iterator of (filename, state, mode, size, mtime) tuples
603 602
604 603 `all`: also include with `state == b' '` dirstate tree nodes that
605 604 don't have an associated `DirstateItem`.
606 605
607 606 """
608 607 return self._map.debug_iter(all)
609 608
610 609 def clear(self):
611 610 self._map.clear()
612 611 self.setparents(
613 612 self._nodeconstants.nullid, self._nodeconstants.nullid
614 613 )
615 614 util.clearcachedproperty(self, b"_dirs")
616 615 util.clearcachedproperty(self, b"_alldirs")
617 616 util.clearcachedproperty(self, b"dirfoldmap")
618 617
619 618 def items(self):
620 619 return self._map.items()
621 620
622 621 # forward for python2,3 compat
623 622 iteritems = items
624 623
625 624 def keys(self):
626 625 return iter(self._map)
627 626
628 627 ### reading/setting parents
629 628
630 629 def setparents(self, p1, p2, fold_p2=False):
631 630 self._parents = (p1, p2)
632 631 self._dirtyparents = True
633 632 copies = {}
634 633 if fold_p2:
635 634 copies = self._map.setparents_fixup()
636 635 return copies
637 636
638 637 ### disk interaction
639 638
640 639 @propertycache
641 640 def identity(self):
642 641 self._map
643 642 return self.identity
644 643
645 644 def write(self, tr, st):
646 645 if not self._use_dirstate_v2:
647 646 p1, p2 = self.parents()
648 647 packed = self._map.write_v1(p1, p2)
649 648 st.write(packed)
650 649 st.close()
651 650 self._dirtyparents = False
652 651 return
653 652
654 653 # We can only append to an existing data file if there is one
655 654 write_mode = self._write_mode
656 655 if self.docket.uuid is None:
657 656 write_mode = WRITE_MODE_FORCE_NEW
658 657 packed, meta, append = self._map.write_v2(write_mode)
659 658 if append:
660 659 docket = self.docket
661 660 data_filename = docket.data_filename()
662 661 with self._opener(data_filename, b'r+b') as fp:
663 662 fp.seek(docket.data_size)
664 663 assert fp.tell() == docket.data_size
665 664 written = fp.write(packed)
666 665 if written is not None: # py2 may return None
667 666 assert written == len(packed), (written, len(packed))
668 667 docket.data_size += len(packed)
669 668 docket.parents = self.parents()
670 669 docket.tree_metadata = meta
671 670 st.write(docket.serialize())
672 671 st.close()
673 672 else:
674 673 self.write_v2_no_append(tr, st, meta, packed)
675 674 # Reload from the newly-written file
676 675 util.clearcachedproperty(self, b"_map")
677 676 self._dirtyparents = False
678 677
679 678 ### code related to maintaining and accessing "extra" property
680 679 # (e.g. "has_dir")
681 680
682 681 @propertycache
683 682 def filefoldmap(self):
684 683 """Returns a dictionary mapping normalized case paths to their
685 684 non-normalized versions.
686 685 """
687 686 return self._map.filefoldmapasdict()
688 687
689 688 def hastrackeddir(self, d):
690 689 return self._map.hastrackeddir(d)
691 690
692 691 def hasdir(self, d):
693 692 return self._map.hasdir(d)
694 693
695 694 @propertycache
696 695 def dirfoldmap(self):
697 696 f = {}
698 697 normcase = util.normcase
699 698 for name in self._map.tracked_dirs():
700 699 f[normcase(name)] = name
701 700 return f
702 701
703 702 ### code related to manipulation of entries and copy-sources
704 703
705 704 def set_tracked(self, f):
706 705 return self._map.set_tracked(f)
707 706
708 707 def set_untracked(self, f):
709 708 return self._map.set_untracked(f)
710 709
711 710 def set_clean(self, filename, mode, size, mtime):
712 711 self._map.set_clean(filename, mode, size, mtime)
713 712
714 713 def set_possibly_dirty(self, f):
715 714 self._map.set_possibly_dirty(f)
716 715
717 716 def reset_state(
718 717 self,
719 718 filename,
720 719 wc_tracked=False,
721 720 p1_tracked=False,
722 721 p2_info=False,
723 722 has_meaningful_mtime=True,
724 723 parentfiledata=None,
725 724 ):
726 725 return self._map.reset_state(
727 726 filename,
728 727 wc_tracked,
729 728 p1_tracked,
730 729 p2_info,
731 730 has_meaningful_mtime,
732 731 parentfiledata,
733 732 )
@@ -1,569 +1,584
1 1 ==============================================================================
2 2 Check potential race conditions between a dirstate's read and other operations
3 3 ==============================================================================
4 4
5 5 #testcases dirstate-v1 dirstate-v2-append dirstate-v2-rewrite
6 6 #testcases pre-all-read pre-some-read
7 7
8 8 Some commands, like `hg status`, do not need to take the wlock but need to
9 9 access dirstate data.
10 10 Other commands might update the dirstate data while this happens.
11 11
12 12 This can create issues if repository data is read in the wrong order, or for
13 13 the dirstate-v2 format where the data is contained in multiple files.
14 14
15 15 This test file is meant to test various cases where such parallel operations
16 16 happen and make sure the reading process behaves fine. We do so with a `hg
17 17 status` command since it is probably the most advanced of such read-only
18 18 command.
19 19
20 20 It bears simililarity with `tests/test-dirstate-status-race.t ` but tests a
21 21 different type of race.
22 22
23 23 Setup
24 24 =====
25 25
26 26 $ cat >> $HGRCPATH << EOF
27 27 > [storage]
28 28 > dirstate-v2.slow-path=allow
29 29 > EOF
30 30
31 31 #if no-dirstate-v1
32 32 $ cat >> $HGRCPATH << EOF
33 33 > [format]
34 34 > use-dirstate-v2=yes
35 35 > EOF
36 36 #else
37 37 $ cat >> $HGRCPATH << EOF
38 38 > [format]
39 39 > use-dirstate-v2=no
40 40 > EOF
41 41 #endif
42 42
43 43 #if dirstate-v2-rewrite
44 44 $ d2args="--config devel.dirstate.v2.data_update_mode=force-new"
45 45 #endif
46 46 #if dirstate-v2-append
47 47 $ d2args="--config devel.dirstate.v2.data_update_mode=force-append"
48 48 #endif
49 49
50 50
51 51 #if dirstate-v1
52 52 $ cfg="devel.sync.dirstate.pre-read-file"
53 53 #else
54 54 #if pre-all-read
55 55 $ cfg="devel.sync.dirstate.pre-read-file"
56 56 #else
57 57 $ cfg="devel.sync.dirstate.post-docket-read-file"
58 58 #endif
59 59 #endif
60 60
61 61 $ directories="dir dir/nested dir2"
62 62 $ first_files="dir/nested/a dir/b dir/c dir/d dir2/e f"
63 63 $ second_files="g dir/nested/h dir/i dir/j dir2/k dir2/l dir/nested/m"
64 64 $ extra_files="dir/n dir/o p q"
65 65
66 66 $ hg init reference-repo
67 67 $ cd reference-repo
68 68 $ mkdir -p dir/nested dir2
69 69 $ touch -t 200001010000 $first_files $directories
70 70 $ hg commit -Aqm "recreate a bunch of files to facilitate dirstate-v2 append"
71 71 $ touch -t 200001010010 $second_files $directories
72 72 $ hg commit -Aqm "more files to have two commit"
73 73 $ hg log -G -v
74 74 @ changeset: 1:9a86dcbfb938
75 75 | tag: tip
76 76 | user: test
77 77 | date: Thu Jan 01 00:00:00 1970 +0000
78 78 | files: dir/i dir/j dir/nested/h dir/nested/m dir2/k dir2/l g
79 79 | description:
80 80 | more files to have two commit
81 81 |
82 82 |
83 83 o changeset: 0:4f23db756b09
84 84 user: test
85 85 date: Thu Jan 01 00:00:00 1970 +0000
86 86 files: dir/b dir/c dir/d dir/nested/a dir2/e f
87 87 description:
88 88 recreate a bunch of files to facilitate dirstate-v2 append
89 89
90 90
91 91 $ hg manifest
92 92 dir/b
93 93 dir/c
94 94 dir/d
95 95 dir/i
96 96 dir/j
97 97 dir/nested/a
98 98 dir/nested/h
99 99 dir/nested/m
100 100 dir2/e
101 101 dir2/k
102 102 dir2/l
103 103 f
104 104 g
105 105
106 106 Add some unknown files and refresh the dirstate
107 107
108 108 $ touch -t 200001010020 $extra_files
109 109 $ hg add dir/o
110 110 $ hg remove dir/nested/m
111 111
112 112 $ hg st --config devel.dirstate.v2.data_update_mode=force-new
113 113 A dir/o
114 114 R dir/nested/m
115 115 ? dir/n
116 116 ? p
117 117 ? q
118 118 $ hg debugstate
119 119 n 644 0 2000-01-01 00:00:00 dir/b
120 120 n 644 0 2000-01-01 00:00:00 dir/c
121 121 n 644 0 2000-01-01 00:00:00 dir/d
122 122 n 644 0 2000-01-01 00:10:00 dir/i
123 123 n 644 0 2000-01-01 00:10:00 dir/j
124 124 n 644 0 2000-01-01 00:00:00 dir/nested/a
125 125 n 644 0 2000-01-01 00:10:00 dir/nested/h
126 126 r ?????????????????????????????????? dir/nested/m (glob)
127 127 a ?????????????????????????????????? dir/o (glob)
128 128 n 644 0 2000-01-01 00:00:00 dir2/e
129 129 n 644 0 2000-01-01 00:10:00 dir2/k
130 130 n 644 0 2000-01-01 00:10:00 dir2/l
131 131 n 644 0 2000-01-01 00:00:00 f
132 132 n 644 0 2000-01-01 00:10:00 g
133 133 $ hg debugstate > ../reference
134 134 $ cd ..
135 135
136 136 Actual Testing
137 137 ==============
138 138
139 139 Race with a `hg add`
140 140 -------------------
141 141
142 142 $ cp -a reference-repo race-with-add
143 143 $ cd race-with-add
144 144
145 145 spin a `hg status` with some caches to update
146 146
147 147 $ hg st >$TESTTMP/status-race-lock.out 2>$TESTTMP/status-race-lock.log \
148 148 > --config rhg.on-unsupported=abort \
149 149 > --config ${cfg}=$TESTTMP/status-race-lock \
150 150 > &
151 151 $ $RUNTESTDIR/testlib/wait-on-file 5 $TESTTMP/status-race-lock.waiting
152 152
153 153 Add a file
154 154
155 155 $ hg $d2args add dir/n
156 156 $ touch $TESTTMP/status-race-lock
157 157 $ wait
158 158
159 159 The file should in a "added" state
160 160
161 161 $ hg status
162 162 A dir/n
163 163 A dir/o
164 164 R dir/nested/m
165 165 ? p
166 166 ? q
167 167
168 168 The status process should return a consistent result and not crash.
169 169
170 170 #if dirstate-v1
171 171 $ cat $TESTTMP/status-race-lock.out
172 172 A dir/n
173 173 A dir/o
174 174 R dir/nested/m
175 175 ? p
176 176 ? q
177 177 $ cat $TESTTMP/status-race-lock.log
178 178 #else
179 179 #if rhg
180 180 #if pre-all-read
181 181 $ cat $TESTTMP/status-race-lock.out
182 182 A dir/n
183 183 A dir/o
184 184 R dir/nested/m
185 185 ? p
186 186 ? q
187 187 $ cat $TESTTMP/status-race-lock.log
188 188 #else
189 189 #if dirstate-v2-append
190 190 $ cat $TESTTMP/status-race-lock.out
191 191 A dir/o
192 192 R dir/nested/m
193 193 ? dir/n
194 194 ? p
195 195 ? q
196 196 $ cat $TESTTMP/status-race-lock.log
197 197 #else
198 198 $ cat $TESTTMP/status-race-lock.out
199 199 $ cat $TESTTMP/status-race-lock.log
200 200 abort: dirstate-v2 parse error: not enough bytes on disk
201 201 #endif
202 202 #endif
203 203 #else
204 204 #if rust
205 205 #if dirstate-v2-rewrite
206 206 $ cat $TESTTMP/status-race-lock.out
207 A dir/n
208 A dir/o
209 R dir/nested/m
210 ? p
211 ? q
207 212 $ cat $TESTTMP/status-race-lock.log
208 abort: $ENOENT$: '$TESTTMP/race-with-add/.hg/dirstate.* (glob)
209 213 #else
210 214 $ cat $TESTTMP/status-race-lock.out
211 215 A dir/o
212 216 R dir/nested/m
213 217 ? dir/n
214 218 ? p
215 219 ? q
216 220 $ cat $TESTTMP/status-race-lock.log
217 221 #endif
218 222 #else
219 223 $ cat $TESTTMP/status-race-lock.out
220 224 A dir/n
221 225 A dir/o
222 226 R dir/nested/m
223 227 ? p
224 228 ? q
225 229 $ cat $TESTTMP/status-race-lock.log
226 230 #endif
227 231 #endif
228 232 #endif
229 233
230 234 final cleanup
231 235
232 236 $ rm $TESTTMP/status-race-lock $TESTTMP/status-race-lock.waiting
233 237 $ cd ..
234 238
235 239 Race with a `hg commit`
236 240 -----------------------
237 241
238 242 $ cp -a reference-repo race-with-commit
239 243 $ cd race-with-commit
240 244
241 245 spin a `hg status with some cache to update
242 246
243 247 $ hg st >$TESTTMP/status-race-lock.out 2>$TESTTMP/status-race-lock.log \
244 248 > --config rhg.on-unsupported=abort \
245 249 > --config ${cfg}=$TESTTMP/status-race-lock \
246 250 > &
247 251 $ $RUNTESTDIR/testlib/wait-on-file 5 $TESTTMP/status-race-lock.waiting
248 252
249 253 Add a do a commit
250 254
251 255 $ hg status
252 256 A dir/o
253 257 R dir/nested/m
254 258 ? dir/n
255 259 ? p
256 260 ? q
257 261 $ hg $d2args commit -m 'racing commit'
258 262 $ touch $TESTTMP/status-race-lock
259 263 $ wait
260 264
261 265 commit was created, and status is now clean
262 266
263 267 $ hg log -GT '{node|short} {desc}\n'
264 268 @ 02a67a77ee9b racing commit
265 269 |
266 270 o 9a86dcbfb938 more files to have two commit
267 271 |
268 272 o 4f23db756b09 recreate a bunch of files to facilitate dirstate-v2 append
269 273
270 274 $ hg status
271 275 ? dir/n
272 276 ? p
273 277 ? q
274 278
275 279 The status process should return a consistent result and not crash.
276 280
277 281 #if dirstate-v1
278 282 $ cat $TESTTMP/status-race-lock.out
279 283 M dir/o (no-rhg !)
280 284 ? dir/n
281 285 ? p
282 286 ? q
283 287 $ cat $TESTTMP/status-race-lock.log
284 288 warning: ignoring unknown working parent 02a67a77ee9b! (no-rhg !)
285 289 #else
286 290 #if rhg
287 291 #if pre-all-read
288 292 $ cat $TESTTMP/status-race-lock.out
289 293 ? dir/n
290 294 ? p
291 295 ? q
292 296 $ cat $TESTTMP/status-race-lock.log
293 297 #else
294 298 #if dirstate-v2-append
295 299 $ cat $TESTTMP/status-race-lock.out
296 300 A dir/o
297 301 R dir/nested/m
298 302 ? dir/n
299 303 ? p
300 304 ? q
301 305 $ cat $TESTTMP/status-race-lock.log
302 306 #else
303 307 $ cat $TESTTMP/status-race-lock.out
304 308 $ cat $TESTTMP/status-race-lock.log
305 309 abort: dirstate-v2 parse error: not enough bytes on disk
306 310 #endif
307 311 #endif
308 312 #else
309 313 #if rust
310 314 #if dirstate-v2-rewrite
311 315 $ cat $TESTTMP/status-race-lock.out
316 M dir/o
317 ? dir/n
318 ? p
319 ? q
312 320 $ cat $TESTTMP/status-race-lock.log
313 abort: $ENOENT$: '$TESTTMP/race-with-commit/.hg/dirstate.* (glob)
321 warning: ignoring unknown working parent 02a67a77ee9b!
314 322 #else
315 323 $ cat $TESTTMP/status-race-lock.out
316 324 A dir/o
317 325 R dir/nested/m
318 326 ? dir/n
319 327 ? p
320 328 ? q
321 329 $ cat $TESTTMP/status-race-lock.log
322 330 #endif
323 331 #else
324 332 $ cat $TESTTMP/status-race-lock.out
325 333 M dir/o
326 334 ? dir/n
327 335 ? p
328 336 ? q
329 337 $ cat $TESTTMP/status-race-lock.log
330 338 warning: ignoring unknown working parent 02a67a77ee9b!
331 339 #endif
332 340 #endif
333 341 #endif
334 342
335 343 final cleanup
336 344
337 345 $ rm $TESTTMP/status-race-lock $TESTTMP/status-race-lock.waiting
338 346 $ cd ..
339 347
340 348 Race with a `hg update`
341 349 -----------------------
342 350
343 351 $ cp -a reference-repo race-with-update
344 352 $ cd race-with-update
345 353
346 354 spin a `hg status` with some caches to update
347 355
348 356 $ hg st >$TESTTMP/status-race-lock.out 2>$TESTTMP/status-race-lock.log \
349 357 > --config rhg.on-unsupported=abort \
350 358 > --config ${cfg}=$TESTTMP/status-race-lock \
351 359 > &
352 360 $ $RUNTESTDIR/testlib/wait-on-file 5 $TESTTMP/status-race-lock.waiting
353 361 do an update
354 362
355 363 $ hg status
356 364 A dir/o
357 365 R dir/nested/m
358 366 ? dir/n
359 367 ? p
360 368 ? q
361 369 $ hg log -GT '{node|short} {desc}\n'
362 370 @ 9a86dcbfb938 more files to have two commit
363 371 |
364 372 o 4f23db756b09 recreate a bunch of files to facilitate dirstate-v2 append
365 373
366 374 $ hg $d2args update --merge ".~1"
367 375 0 files updated, 0 files merged, 6 files removed, 0 files unresolved
368 376 $ touch $TESTTMP/status-race-lock
369 377 $ wait
370 378 #if rhg dirstate-v2-append pre-some-read
371 379 $ hg log -GT '{node|short} {desc}\n'
372 380 @ 9a86dcbfb938 more files to have two commit
373 381 |
374 382 o 4f23db756b09 recreate a bunch of files to facilitate dirstate-v2 append
375 383
376 384 $ hg status
377 385 A dir/o
378 386 R dir/nested/m
379 387 ! dir/i
380 388 ! dir/j
381 389 ! dir/nested/h
382 390 ! dir2/k
383 391 ! dir2/l
384 392 ! g
385 393 ? dir/n
386 394 ? p
387 395 ? q
388 396 #else
389 397 $ hg log -GT '{node|short} {desc}\n'
390 398 o 9a86dcbfb938 more files to have two commit
391 399 |
392 400 @ 4f23db756b09 recreate a bunch of files to facilitate dirstate-v2 append
393 401
394 402 $ hg status
395 403 A dir/o
396 404 ? dir/n
397 405 ? p
398 406 ? q
399 407 #endif
400 408
401 409 The status process should return a consistent result and not crash.
402 410
403 411 #if dirstate-v1
404 412 $ cat $TESTTMP/status-race-lock.out
405 413 A dir/o
406 414 ? dir/n
407 415 ? p
408 416 ? q
409 417 $ cat $TESTTMP/status-race-lock.log
410 418 #else
411 419 #if rhg
412 420 #if pre-all-read
413 421 $ cat $TESTTMP/status-race-lock.out
414 422 A dir/o
415 423 ? dir/n
416 424 ? p
417 425 ? q
418 426 $ cat $TESTTMP/status-race-lock.log
419 427 #else
420 428 #if dirstate-v2-append
421 429 $ cat $TESTTMP/status-race-lock.out
422 430 A dir/o
423 431 R dir/nested/m
424 432 ! dir/i
425 433 ! dir/j
426 434 ! dir/nested/h
427 435 ! dir2/k
428 436 ! dir2/l
429 437 ! g
430 438 ? dir/n
431 439 ? p
432 440 ? q
433 441 $ cat $TESTTMP/status-race-lock.log
434 442 #else
435 443 $ cat $TESTTMP/status-race-lock.out
436 444 $ cat $TESTTMP/status-race-lock.log
437 445 abort: dirstate-v2 parse error: not enough bytes on disk
438 446 #endif
439 447 #endif
440 448 #else
441 449 #if rust
442 450 #if dirstate-v2-rewrite
443 451 $ cat $TESTTMP/status-race-lock.out
452 A dir/o
453 ? dir/n
454 ? p
455 ? q
444 456 $ cat $TESTTMP/status-race-lock.log
445 abort: $ENOENT$: '$TESTTMP/race-with-update/.hg/dirstate.* (glob)
446 457 #else
447 458 $ cat $TESTTMP/status-race-lock.out
448 459 A dir/o
449 460 R dir/nested/m
450 461 ! dir/i
451 462 ! dir/j
452 463 ! dir/nested/h
453 464 ! dir2/k
454 465 ! dir2/l
455 466 ! g
456 467 ? dir/n
457 468 ? p
458 469 ? q
459 470 $ cat $TESTTMP/status-race-lock.log
460 471 #endif
461 472 #else
462 473 $ cat $TESTTMP/status-race-lock.out
463 474 A dir/o
464 475 ? dir/n
465 476 ? p
466 477 ? q
467 478 $ cat $TESTTMP/status-race-lock.log
468 479 #endif
469 480 #endif
470 481 #endif
471 482
472 483 final cleanup
473 484
474 485 $ rm $TESTTMP/status-race-lock $TESTTMP/status-race-lock.waiting
475 486 $ cd ..
476 487
477 488 Race with a cache updating `hg status`
478 489 --------------------------------------
479 490
480 491 It is interesting to race with "read-only" operation (that still update its cache)
481 492
482 493 $ cp -a reference-repo race-with-status
483 494 $ cd race-with-status
484 495
485 496 spin a `hg status` with some caches to update
486 497
487 498 $ hg st >$TESTTMP/status-race-lock.out 2>$TESTTMP/status-race-lock.log \
488 499 > --config rhg.on-unsupported=abort \
489 500 > --config ${cfg}=$TESTTMP/status-race-lock \
490 501 > &
491 502 $ $RUNTESTDIR/testlib/wait-on-file 5 $TESTTMP/status-race-lock.waiting
492 503 do an update
493 504
494 505 $ touch -t 200001020006 f
495 506 $ hg $d2args status
496 507 A dir/o
497 508 R dir/nested/m
498 509 ? dir/n
499 510 ? p
500 511 ? q
501 512 $ touch $TESTTMP/status-race-lock
502 513 $ wait
503 514
504 515 The status process should return a consistent result and not crash.
505 516
506 517 #if dirstate-v1
507 518 $ cat $TESTTMP/status-race-lock.out
508 519 A dir/o
509 520 R dir/nested/m
510 521 ? dir/n
511 522 ? p
512 523 ? q
513 524 $ cat $TESTTMP/status-race-lock.log
514 525 #else
515 526 #if rhg
516 527 #if pre-all-read
517 528 $ cat $TESTTMP/status-race-lock.out
518 529 A dir/o
519 530 R dir/nested/m
520 531 ? dir/n
521 532 ? p
522 533 ? q
523 534 $ cat $TESTTMP/status-race-lock.log
524 535 #else
525 536 #if dirstate-v2-append
526 537 $ cat $TESTTMP/status-race-lock.out
527 538 A dir/o
528 539 R dir/nested/m
529 540 ? dir/n
530 541 ? p
531 542 ? q
532 543 $ cat $TESTTMP/status-race-lock.log
533 544 #else
534 545 $ cat $TESTTMP/status-race-lock.out
535 546 $ cat $TESTTMP/status-race-lock.log
536 547 abort: dirstate-v2 parse error: not enough bytes on disk
537 548 #endif
538 549 #endif
539 550 #else
540 551 #if rust
541 552 #if dirstate-v2-rewrite
542 553 $ cat $TESTTMP/status-race-lock.out
554 A dir/o
555 R dir/nested/m
556 ? dir/n
557 ? p
558 ? q
543 559 $ cat $TESTTMP/status-race-lock.log
544 abort: $ENOENT$: '$TESTTMP/race-with-status/.hg/dirstate.* (glob)
545 560 #else
546 561 $ cat $TESTTMP/status-race-lock.out
547 562 A dir/o
548 563 R dir/nested/m
549 564 ? dir/n
550 565 ? p
551 566 ? q
552 567 $ cat $TESTTMP/status-race-lock.log
553 568 #endif
554 569 #else
555 570 $ cat $TESTTMP/status-race-lock.out
556 571 A dir/o
557 572 R dir/nested/m
558 573 ? dir/n
559 574 ? p
560 575 ? q
561 576 $ cat $TESTTMP/status-race-lock.log
562 577 #endif
563 578 #endif
564 579 #endif
565 580
566 581 final cleanup
567 582
568 583 $ rm $TESTTMP/status-race-lock $TESTTMP/status-race-lock.waiting
569 584 $ cd ..
General Comments 0
You need to be logged in to leave comments. Login now