##// END OF EJS Templates
dirstate: remove need_delay logic...
Raphaël Gomès -
r49215:2b5d1618 default draft
parent child Browse files
Show More
@@ -320,21 +320,6 b' static PyObject *dirstate_item_v1_mtime('
320 320 return PyInt_FromLong(dirstate_item_c_v1_mtime(self));
321 321 };
322 322
323 static PyObject *dirstate_item_need_delay(dirstateItemObject *self,
324 PyObject *now)
325 {
326 int now_s;
327 int now_ns;
328 if (!PyArg_ParseTuple(now, "ii", &now_s, &now_ns)) {
329 return NULL;
330 }
331 if (dirstate_item_c_v1_state(self) == 'n' && self->mtime_s == now_s) {
332 Py_RETURN_TRUE;
333 } else {
334 Py_RETURN_FALSE;
335 }
336 };
337
338 323 static PyObject *dirstate_item_mtime_likely_equal_to(dirstateItemObject *self,
339 324 PyObject *other)
340 325 {
@@ -548,8 +533,6 b' static PyMethodDef dirstate_item_methods'
548 533 "return a \"size\" suitable for v1 serialization"},
549 534 {"v1_mtime", (PyCFunction)dirstate_item_v1_mtime, METH_NOARGS,
550 535 "return a \"mtime\" suitable for v1 serialization"},
551 {"need_delay", (PyCFunction)dirstate_item_need_delay, METH_O,
552 "True if the stored mtime would be ambiguous with the current time"},
553 536 {"mtime_likely_equal_to", (PyCFunction)dirstate_item_mtime_likely_equal_to,
554 537 METH_O, "True if the stored mtime is likely equal to the given mtime"},
555 538 {"from_v1_data", (PyCFunction)dirstate_item_from_v1_meth,
@@ -922,12 +905,9 b' static PyObject *pack_dirstate(PyObject '
922 905 Py_ssize_t nbytes, pos, l;
923 906 PyObject *k, *v = NULL, *pn;
924 907 char *p, *s;
925 int now_s;
926 int now_ns;
927 908
928 if (!PyArg_ParseTuple(args, "O!O!O!(ii):pack_dirstate", &PyDict_Type,
929 &map, &PyDict_Type, &copymap, &PyTuple_Type, &pl,
930 &now_s, &now_ns)) {
909 if (!PyArg_ParseTuple(args, "O!O!O!:pack_dirstate", &PyDict_Type, &map,
910 &PyDict_Type, &copymap, &PyTuple_Type, &pl)) {
931 911 return NULL;
932 912 }
933 913
@@ -996,21 +976,6 b' static PyObject *pack_dirstate(PyObject '
996 976 mode = dirstate_item_c_v1_mode(tuple);
997 977 size = dirstate_item_c_v1_size(tuple);
998 978 mtime = dirstate_item_c_v1_mtime(tuple);
999 if (state == 'n' && tuple->mtime_s == now_s) {
1000 /* See pure/parsers.py:pack_dirstate for why we do
1001 * this. */
1002 mtime = -1;
1003 mtime_unset = (PyObject *)dirstate_item_from_v1_data(
1004 state, mode, size, mtime);
1005 if (!mtime_unset) {
1006 goto bail;
1007 }
1008 if (PyDict_SetItem(map, k, mtime_unset) == -1) {
1009 goto bail;
1010 }
1011 Py_DECREF(mtime_unset);
1012 mtime_unset = NULL;
1013 }
1014 979 *p++ = state;
1015 980 putbe32((uint32_t)mode, p);
1016 981 putbe32((uint32_t)size, p + 4);
@@ -779,25 +779,6 b' class dirstate(object):'
779 779 # filesystem's notion of 'now'
780 780 now = timestamp.mtime_of(util.fstat(st))
781 781
782 # enough 'delaywrite' prevents 'pack_dirstate' from dropping
783 # timestamp of each entries in dirstate, because of 'now > mtime'
784 delaywrite = self._ui.configint(b'debug', b'dirstate.delaywrite')
785 if delaywrite > 0:
786 # do we have any files to delay for?
787 for f, e in pycompat.iteritems(self._map):
788 if e.need_delay(now):
789 import time # to avoid useless import
790
791 # rather than sleep n seconds, sleep until the next
792 # multiple of n seconds
793 clock = time.time()
794 start = int(clock) - (int(clock) % delaywrite)
795 end = start + delaywrite
796 time.sleep(end - clock)
797 # trust our estimate that the end is near now
798 now = timestamp.timestamp((end, 0))
799 break
800
801 782 self._map.write(tr, st, now)
802 783 self._dirty = False
803 784
@@ -446,11 +446,11 b' class dirstatemap(_dirstatemapcommon):'
446 446
447 447 def write(self, tr, st, now):
448 448 if self._use_dirstate_v2:
449 packed, meta = v2.pack_dirstate(self._map, self.copymap, now)
449 packed, meta = v2.pack_dirstate(self._map, self.copymap)
450 450 self.write_v2_no_append(tr, st, meta, packed)
451 451 else:
452 452 packed = parsers.pack_dirstate(
453 self._map, self.copymap, self.parents(), now
453 self._map, self.copymap, self.parents()
454 454 )
455 455 st.write(packed)
456 456 st.close()
@@ -658,7 +658,7 b' if rustmod is not None:'
658 658 def write(self, tr, st, now):
659 659 if not self._use_dirstate_v2:
660 660 p1, p2 = self.parents()
661 packed = self._map.write_v1(p1, p2, now)
661 packed = self._map.write_v1(p1, p2)
662 662 st.write(packed)
663 663 st.close()
664 664 self._dirtyparents = False
@@ -666,7 +666,7 b' if rustmod is not None:'
666 666
667 667 # We can only append to an existing data file if there is one
668 668 can_append = self.docket.uuid is not None
669 packed, meta, append = self._map.write_v2(now, can_append)
669 packed, meta, append = self._map.write_v2(can_append)
670 670 if append:
671 671 docket = self.docket
672 672 data_filename = docket.data_filename()
@@ -174,12 +174,10 b' class Node(object):'
174 174 )
175 175
176 176
177 def pack_dirstate(map, copy_map, now):
177 def pack_dirstate(map, copy_map):
178 178 """
179 179 Pack `map` and `copy_map` into the dirstate v2 binary format and return
180 180 the bytearray.
181 `now` is a timestamp of the current filesystem time used to detect race
182 conditions in writing the dirstate to disk, see inline comment.
183 181
184 182 The on-disk format expects a tree-like structure where the leaves are
185 183 written first (and sorted per-directory), going up levels until the root
@@ -284,17 +282,6 b' def pack_dirstate(map, copy_map, now):'
284 282 stack.append(current_node)
285 283
286 284 for index, (path, entry) in enumerate(sorted_map, 1):
287 if entry.need_delay(now):
288 # The file was last modified "simultaneously" with the current
289 # write to dirstate (i.e. within the same second for file-
290 # systems with a granularity of 1 sec). This commonly happens
291 # for at least a couple of files on 'update'.
292 # The user could change the file without changing its size
293 # within the same second. Invalidate the file's mtime in
294 # dirstate, forcing future 'status' calls to compare the
295 # contents of the file if the size is the same. This prevents
296 # mistakenly treating such files as clean.
297 entry.set_possibly_dirty()
298 285 nodes_with_entry_count += 1
299 286 if path in copy_map:
300 287 nodes_with_copy_source_count += 1
@@ -536,10 +536,6 b' class DirstateItem(object):'
536 536 else:
537 537 return self._mtime_s
538 538
539 def need_delay(self, now):
540 """True if the stored mtime would be ambiguous with the current time"""
541 return self.v1_state() == b'n' and self._mtime_s == now[0]
542
543 539
544 540 def gettype(q):
545 541 return int(q & 0xFFFF)
@@ -905,23 +901,11 b' def parse_dirstate(dmap, copymap, st):'
905 901 return parents
906 902
907 903
908 def pack_dirstate(dmap, copymap, pl, now):
904 def pack_dirstate(dmap, copymap, pl):
909 905 cs = stringio()
910 906 write = cs.write
911 907 write(b"".join(pl))
912 908 for f, e in pycompat.iteritems(dmap):
913 if e.need_delay(now):
914 # The file was last modified "simultaneously" with the current
915 # write to dirstate (i.e. within the same second for file-
916 # systems with a granularity of 1 sec). This commonly happens
917 # for at least a couple of files on 'update'.
918 # The user could change the file without changing its size
919 # within the same second. Invalidate the file's mtime in
920 # dirstate, forcing future 'status' calls to compare the
921 # contents of the file if the size is the same. This prevents
922 # mistakenly treating such files as clean.
923 e.set_possibly_dirty()
924
925 909 if f in copymap:
926 910 f = b"%s\0%s" % (f, copymap[f])
927 911 e = _pack(
@@ -590,16 +590,6 b' impl DirstateEntry {'
590 590 pub fn debug_tuple(&self) -> (u8, i32, i32, i32) {
591 591 (self.state().into(), self.mode(), self.size(), self.mtime())
592 592 }
593
594 /// True if the stored mtime would be ambiguous with the current time
595 pub fn need_delay(&self, now: TruncatedTimestamp) -> bool {
596 if let Some(mtime) = self.mtime {
597 self.state() == EntryState::Normal
598 && mtime.truncated_seconds() == now.truncated_seconds()
599 } else {
600 false
601 }
602 }
603 593 }
604 594
605 595 impl EntryState {
@@ -677,25 +677,6 b" impl<'on_disk> DirstateMap<'on_disk> {"
677 677 })
678 678 }
679 679
680 fn clear_known_ambiguous_mtimes(
681 &mut self,
682 paths: &[impl AsRef<HgPath>],
683 ) -> Result<(), DirstateV2ParseError> {
684 for path in paths {
685 if let Some(node) = Self::get_node_mut(
686 self.on_disk,
687 &mut self.unreachable_bytes,
688 &mut self.root,
689 path.as_ref(),
690 )? {
691 if let NodeData::Entry(entry) = &mut node.data {
692 entry.set_possibly_dirty();
693 }
694 }
695 }
696 Ok(())
697 }
698
699 680 fn count_dropped_path(unreachable_bytes: &mut u32, path: &Cow<HgPath>) {
700 681 if let Cow::Borrowed(path) = path {
701 682 *unreachable_bytes += path.len() as u32
@@ -930,29 +911,20 b' impl OwningDirstateMap {'
930 911 pub fn pack_v1(
931 912 &mut self,
932 913 parents: DirstateParents,
933 now: TruncatedTimestamp,
934 914 ) -> Result<Vec<u8>, DirstateError> {
935 915 let map = self.get_map_mut();
936 let mut ambiguous_mtimes = Vec::new();
937 916 // Optizimation (to be measured?): pre-compute size to avoid `Vec`
938 917 // reallocations
939 918 let mut size = parents.as_bytes().len();
940 919 for node in map.iter_nodes() {
941 920 let node = node?;
942 if let Some(entry) = node.entry()? {
921 if node.entry()?.is_some() {
943 922 size += packed_entry_size(
944 923 node.full_path(map.on_disk)?,
945 924 node.copy_source(map.on_disk)?,
946 925 );
947 if entry.need_delay(now) {
948 ambiguous_mtimes.push(
949 node.full_path_borrowed(map.on_disk)?
950 .detach_from_tree(),
951 )
952 926 }
953 927 }
954 }
955 map.clear_known_ambiguous_mtimes(&ambiguous_mtimes)?;
956 928
957 929 let mut packed = Vec::with_capacity(size);
958 930 packed.extend(parents.as_bytes());
@@ -978,26 +950,9 b' impl OwningDirstateMap {'
978 950 #[timed]
979 951 pub fn pack_v2(
980 952 &mut self,
981 now: TruncatedTimestamp,
982 953 can_append: bool,
983 954 ) -> Result<(Vec<u8>, Vec<u8>, bool), DirstateError> {
984 955 let map = self.get_map_mut();
985 let mut paths = Vec::new();
986 for node in map.iter_nodes() {
987 let node = node?;
988 if let Some(entry) = node.entry()? {
989 if entry.need_delay(now) {
990 paths.push(
991 node.full_path_borrowed(map.on_disk)?
992 .detach_from_tree(),
993 )
994 }
995 }
996 }
997 // Borrow of `self` ends here since we collect cloned paths
998
999 map.clear_known_ambiguous_mtimes(&paths)?;
1000
1001 956 on_disk::write(map, can_append)
1002 957 }
1003 958
@@ -18,7 +18,7 b' use cpython::{'
18 18
19 19 use crate::{
20 20 dirstate::copymap::{CopyMap, CopyMapItemsIterator, CopyMapKeysIterator},
21 dirstate::item::{timestamp, DirstateItem},
21 dirstate::item::DirstateItem,
22 22 pybytes_deref::PyBytesDeref,
23 23 };
24 24 use hg::{
@@ -194,16 +194,13 b' py_class!(pub class DirstateMap |py| {'
194 194 &self,
195 195 p1: PyObject,
196 196 p2: PyObject,
197 now: (u32, u32)
198 197 ) -> PyResult<PyBytes> {
199 let now = timestamp(py, now)?;
200
201 198 let mut inner = self.inner(py).borrow_mut();
202 199 let parents = DirstateParents {
203 200 p1: extract_node_id(py, &p1)?,
204 201 p2: extract_node_id(py, &p2)?,
205 202 };
206 let result = inner.pack_v1(parents, now);
203 let result = inner.pack_v1(parents);
207 204 match result {
208 205 Ok(packed) => Ok(PyBytes::new(py, &packed)),
209 206 Err(_) => Err(PyErr::new::<exc::OSError, _>(
@@ -218,13 +215,10 b' py_class!(pub class DirstateMap |py| {'
218 215 /// instead of written to a new data file (False).
219 216 def write_v2(
220 217 &self,
221 now: (u32, u32),
222 218 can_append: bool,
223 219 ) -> PyResult<PyObject> {
224 let now = timestamp(py, now)?;
225
226 220 let mut inner = self.inner(py).borrow_mut();
227 let result = inner.pack_v2(now, can_append);
221 let result = inner.pack_v2(can_append);
228 222 match result {
229 223 Ok((packed, tree_metadata, append)) => {
230 224 let packed = PyBytes::new(py, &packed);
@@ -194,11 +194,6 b' py_class!(pub class DirstateItem |py| {'
194 194 Ok(mtime)
195 195 }
196 196
197 def need_delay(&self, now: (u32, u32)) -> PyResult<bool> {
198 let now = timestamp(py, now)?;
199 Ok(self.entry(py).get().need_delay(now))
200 }
201
202 197 def mtime_likely_equal_to(&self, other: (u32, u32)) -> PyResult<bool> {
203 198 if let Some(mtime) = self.entry(py).get().truncated_mtime() {
204 199 Ok(mtime.likely_equal(timestamp(py, other)?))
@@ -37,14 +37,8 b" parsers = policy.importmod('parsers')"
37 37 has_rust_dirstate = policy.importrust('dirstate') is not None
38 38
39 39
40 def pack_dirstate(fakenow, orig, dmap, copymap, pl, now):
41 # execute what original parsers.pack_dirstate should do actually
42 # for consistency
43 for f, e in dmap.items():
44 if e.need_delay(now):
45 e.set_possibly_dirty()
46
47 return orig(dmap, copymap, pl, fakenow)
40 def pack_dirstate(orig, dmap, copymap, pl):
41 return orig(dmap, copymap, pl)
48 42
49 43
50 44 def fakewrite(ui, func):
@@ -67,19 +61,19 b' def fakewrite(ui, func):'
67 61 # The Rust implementation does not use public parse/pack dirstate
68 62 # to prevent conversion round-trips
69 63 orig_dirstatemap_write = dirstatemapmod.dirstatemap.write
70 wrapper = lambda self, tr, st, now: orig_dirstatemap_write(
71 self, tr, st, fakenow
72 )
64 wrapper = lambda self, tr, st: orig_dirstatemap_write(self, tr, st)
73 65 dirstatemapmod.dirstatemap.write = wrapper
74 66
75 67 orig_get_fs_now = timestamp.get_fs_now
76 wrapper = lambda *args: pack_dirstate(fakenow, orig_pack_dirstate, *args)
68 wrapper = lambda *args: pack_dirstate(orig_pack_dirstate, *args)
77 69
78 70 orig_module = parsers
79 71 orig_pack_dirstate = parsers.pack_dirstate
80 72
81 73 orig_module.pack_dirstate = wrapper
82 timestamp.get_fs_now = lambda *args: fakenow
74 timestamp.get_fs_now = (
75 lambda *args: fakenow
76 ) # XXX useless for this purpose now
83 77 try:
84 78 return func()
85 79 finally:
@@ -349,6 +349,10 b' Test that updated files are treated as "'
349 349 aren't changed), even if none of mode, size and timestamp of them
350 350 isn't changed on the filesystem (see also issue4583).
351 351
352 This test is now "best effort" as the mechanism to prevent such race are
353 getting better, it get more complicated to test a specific scenario that would
354 trigger it. If you see flakyness here, there is a race.
355
352 356 $ cat > $TESTTMP/abort.py <<EOF
353 357 > from __future__ import absolute_import
354 358 > # emulate aborting before "recordupdates()". in this case, files
@@ -365,13 +369,6 b" isn't changed on the filesystem (see als"
365 369 > extensions.wrapfunction(merge, "applyupdates", applyupdates)
366 370 > EOF
367 371
368 $ cat >> .hg/hgrc <<EOF
369 > [fakedirstatewritetime]
370 > # emulate invoking dirstate.write() via repo.status()
371 > # at 2000-01-01 00:00
372 > fakenow = 200001010000
373 > EOF
374
375 372 (file gotten from other revision)
376 373
377 374 $ hg update -q -C 2
@@ -381,12 +378,8 b" isn't changed on the filesystem (see als"
381 378 $ hg update -q -C 3
382 379 $ cat b
383 380 This is file b1
384 $ touch -t 200001010000 b
385 $ hg debugrebuildstate
386
387 381 $ cat >> .hg/hgrc <<EOF
388 382 > [extensions]
389 > fakedirstatewritetime = $TESTDIR/fakedirstatewritetime.py
390 383 > abort = $TESTTMP/abort.py
391 384 > EOF
392 385 $ hg merge 5
@@ -394,13 +387,11 b" isn't changed on the filesystem (see als"
394 387 [255]
395 388 $ cat >> .hg/hgrc <<EOF
396 389 > [extensions]
397 > fakedirstatewritetime = !
398 390 > abort = !
399 391 > EOF
400 392
401 393 $ cat b
402 394 THIS IS FILE B5
403 $ touch -t 200001010000 b
404 395 $ hg status -A b
405 396 M b
406 397
@@ -413,12 +404,10 b" isn't changed on the filesystem (see als"
413 404
414 405 $ cat b
415 406 this is file b6
416 $ touch -t 200001010000 b
417 $ hg debugrebuildstate
407 $ hg status
418 408
419 409 $ cat >> .hg/hgrc <<EOF
420 410 > [extensions]
421 > fakedirstatewritetime = $TESTDIR/fakedirstatewritetime.py
422 411 > abort = $TESTTMP/abort.py
423 412 > EOF
424 413 $ hg merge --tool internal:other 5
@@ -426,13 +415,11 b" isn't changed on the filesystem (see als"
426 415 [255]
427 416 $ cat >> .hg/hgrc <<EOF
428 417 > [extensions]
429 > fakedirstatewritetime = !
430 418 > abort = !
431 419 > EOF
432 420
433 421 $ cat b
434 422 THIS IS FILE B5
435 $ touch -t 200001010000 b
436 423 $ hg status -A b
437 424 M b
438 425
@@ -1021,37 +1021,21 b' Issue1977: multirepo push should fail if'
1021 1021
1022 1022 test if untracked file is not overwritten
1023 1023
1024 (this also tests that updated .hgsubstate is treated as "modified",
1025 when 'merge.update()' is aborted before 'merge.recordupdates()', even
1026 if none of mode, size and timestamp of it isn't changed on the
1027 filesystem (see also issue4583))
1024 (this tests also has a change to update .hgsubstate and merge it within the
1025 same second. It should mark is are modified , even if none of mode, size and
1026 timestamp of it isn't changed on the filesystem (see also issue4583))
1028 1027
1029 1028 $ echo issue3276_ok > repo/s/b
1030 1029 $ hg -R repo2 push -f -q
1031 $ touch -t 200001010000 repo/.hgsubstate
1032 1030
1033 $ cat >> repo/.hg/hgrc <<EOF
1034 > [fakedirstatewritetime]
1035 > # emulate invoking dirstate.write() via repo.status()
1036 > # at 2000-01-01 00:00
1037 > fakenow = 200001010000
1038 >
1039 > [extensions]
1040 > fakedirstatewritetime = $TESTDIR/fakedirstatewritetime.py
1041 > EOF
1042 1031 $ hg -R repo update
1043 1032 b: untracked file differs
1044 1033 abort: untracked files in working directory differ from files in requested revision (in subrepository "s")
1045 1034 [255]
1046 $ cat >> repo/.hg/hgrc <<EOF
1047 > [extensions]
1048 > fakedirstatewritetime = !
1049 > EOF
1050 1035
1051 1036 $ cat repo/s/b
1052 1037 issue3276_ok
1053 1038 $ rm repo/s/b
1054 $ touch -t 200001010000 repo/.hgsubstate
1055 1039 $ hg -R repo revert --all
1056 1040 reverting repo/.hgsubstate
1057 1041 reverting subrepo s
General Comments 0
You need to be logged in to leave comments. Login now