##// END OF EJS Templates
upstream » mercurial-mirror
RSS

Clone from https://www.mercurial-scm.org/repo/hg-all

treemanifest: introduce lazy loading of subdirs...
spectral -
r39551:93486cc4 default
parent child Browse files
Show More
Show file before | Show file after | Hide comments
@@ -686,6 +686,7 b' class treemanifest(object):'
686 self._copyfunc = _noop
686 self._copyfunc = _noop
687 self._dirty = False
687 self._dirty = False
688 self._dirs = {}
688 self._dirs = {}
689 self._lazydirs = {}
689 # Using _lazymanifest here is a little slower than plain old dicts
690 # Using _lazymanifest here is a little slower than plain old dicts
690 self._files = {}
691 self._files = {}
691 self._flags = {}
692 self._flags = {}
@@ -699,9 +700,20 b' class treemanifest(object):'
699 def _subpath(self, path):
700 def _subpath(self, path):
700 return self._dir + path
701 return self._dir + path
701
702
703 def _loadalllazy(self):
704 for k, (path, node, readsubtree) in self._lazydirs.iteritems():
705 self._dirs[k] = readsubtree(path, node)
706 self._lazydirs = {}
707
708 def _loadlazy(self, d):
709 path, node, readsubtree = self._lazydirs[d]
710 self._dirs[d] = readsubtree(path, node)
711 del self._lazydirs[d]
712
702 def __len__(self):
713 def __len__(self):
703 self._load()
714 self._load()
704 size = len(self._files)
715 size = len(self._files)
716 self._loadalllazy()
705 for m in self._dirs.values():
717 for m in self._dirs.values():
706 size += m.__len__()
718 size += m.__len__()
707 return size
719 return size
@@ -714,6 +726,7 b' class treemanifest(object):'
714
726
715 def _isempty(self):
727 def _isempty(self):
716 self._load() # for consistency; already loaded by all callers
728 self._load() # for consistency; already loaded by all callers
729 self._loadalllazy()
717 return (not self._files and (not self._dirs or
730 return (not self._files and (not self._dirs or
718 all(m._isempty() for m in self._dirs.values())))
731 all(m._isempty() for m in self._dirs.values())))
719
732
@@ -741,6 +754,7 b' class treemanifest(object):'
741
754
742 def iterentries(self):
755 def iterentries(self):
743 self._load()
756 self._load()
757 self._loadalllazy()
744 for p, n in sorted(itertools.chain(self._dirs.items(),
758 for p, n in sorted(itertools.chain(self._dirs.items(),
745 self._files.items())):
759 self._files.items())):
746 if p in self._files:
760 if p in self._files:
@@ -751,6 +765,7 b' class treemanifest(object):'
751
765
752 def items(self):
766 def items(self):
753 self._load()
767 self._load()
768 self._loadalllazy()
754 for p, n in sorted(itertools.chain(self._dirs.items(),
769 for p, n in sorted(itertools.chain(self._dirs.items(),
755 self._files.items())):
770 self._files.items())):
756 if p in self._files:
771 if p in self._files:
@@ -763,6 +778,7 b' class treemanifest(object):'
763
778
764 def iterkeys(self):
779 def iterkeys(self):
765 self._load()
780 self._load()
781 self._loadalllazy()
766 for p in sorted(itertools.chain(self._dirs, self._files)):
782 for p in sorted(itertools.chain(self._dirs, self._files)):
767 if p in self._files:
783 if p in self._files:
768 yield self._subpath(p)
784 yield self._subpath(p)
@@ -782,8 +798,12 b' class treemanifest(object):'
782 self._load()
798 self._load()
783 dir, subpath = _splittopdir(f)
799 dir, subpath = _splittopdir(f)
784 if dir:
800 if dir:
801 if dir in self._lazydirs:
802 self._loadlazy(dir)
803
785 if dir not in self._dirs:
804 if dir not in self._dirs:
786 return False
805 return False
806
787 return self._dirs[dir].__contains__(subpath)
807 return self._dirs[dir].__contains__(subpath)
788 else:
808 else:
789 return f in self._files
809 return f in self._files
@@ -792,6 +812,9 b' class treemanifest(object):'
792 self._load()
812 self._load()
793 dir, subpath = _splittopdir(f)
813 dir, subpath = _splittopdir(f)
794 if dir:
814 if dir:
815 if dir in self._lazydirs:
816 self._loadlazy(dir)
817
795 if dir not in self._dirs:
818 if dir not in self._dirs:
796 return default
819 return default
797 return self._dirs[dir].get(subpath, default)
820 return self._dirs[dir].get(subpath, default)
@@ -802,6 +825,9 b' class treemanifest(object):'
802 self._load()
825 self._load()
803 dir, subpath = _splittopdir(f)
826 dir, subpath = _splittopdir(f)
804 if dir:
827 if dir:
828 if dir in self._lazydirs:
829 self._loadlazy(dir)
830
805 return self._dirs[dir].__getitem__(subpath)
831 return self._dirs[dir].__getitem__(subpath)
806 else:
832 else:
807 return self._files[f]
833 return self._files[f]
@@ -810,11 +836,14 b' class treemanifest(object):'
810 self._load()
836 self._load()
811 dir, subpath = _splittopdir(f)
837 dir, subpath = _splittopdir(f)
812 if dir:
838 if dir:
839 if dir in self._lazydirs:
840 self._loadlazy(dir)
841
813 if dir not in self._dirs:
842 if dir not in self._dirs:
814 return ''
843 return ''
815 return self._dirs[dir].flags(subpath)
844 return self._dirs[dir].flags(subpath)
816 else:
845 else:
817 if f in self._dirs:
846 if f in self._lazydirs or f in self._dirs:
818 return ''
847 return ''
819 return self._flags.get(f, '')
848 return self._flags.get(f, '')
820
849
@@ -822,6 +851,9 b' class treemanifest(object):'
822 self._load()
851 self._load()
823 dir, subpath = _splittopdir(f)
852 dir, subpath = _splittopdir(f)
824 if dir:
853 if dir:
854 if dir in self._lazydirs:
855 self._loadlazy(dir)
856
825 return self._dirs[dir].find(subpath)
857 return self._dirs[dir].find(subpath)
826 else:
858 else:
827 return self._files[f], self._flags.get(f, '')
859 return self._files[f], self._flags.get(f, '')
@@ -830,6 +862,9 b' class treemanifest(object):'
830 self._load()
862 self._load()
831 dir, subpath = _splittopdir(f)
863 dir, subpath = _splittopdir(f)
832 if dir:
864 if dir:
865 if dir in self._lazydirs:
866 self._loadlazy(dir)
867
833 self._dirs[dir].__delitem__(subpath)
868 self._dirs[dir].__delitem__(subpath)
834 # If the directory is now empty, remove it
869 # If the directory is now empty, remove it
835 if self._dirs[dir]._isempty():
870 if self._dirs[dir]._isempty():
@@ -845,6 +880,8 b' class treemanifest(object):'
845 self._load()
880 self._load()
846 dir, subpath = _splittopdir(f)
881 dir, subpath = _splittopdir(f)
847 if dir:
882 if dir:
883 if dir in self._lazydirs:
884 self._loadlazy(dir)
848 if dir not in self._dirs:
885 if dir not in self._dirs:
849 self._dirs[dir] = treemanifest(self._subpath(dir))
886 self._dirs[dir] = treemanifest(self._subpath(dir))
850 self._dirs[dir].__setitem__(subpath, n)
887 self._dirs[dir].__setitem__(subpath, n)
@@ -865,6 +902,8 b' class treemanifest(object):'
865 self._load()
902 self._load()
866 dir, subpath = _splittopdir(f)
903 dir, subpath = _splittopdir(f)
867 if dir:
904 if dir:
905 if dir in self._lazydirs:
906 self._loadlazy(dir)
868 if dir not in self._dirs:
907 if dir not in self._dirs:
869 self._dirs[dir] = treemanifest(self._subpath(dir))
908 self._dirs[dir] = treemanifest(self._subpath(dir))
870 self._dirs[dir].setflag(subpath, flags)
909 self._dirs[dir].setflag(subpath, flags)
@@ -879,8 +918,12 b' class treemanifest(object):'
879 if self._copyfunc is _noop:
918 if self._copyfunc is _noop:
880 def _copyfunc(s):
919 def _copyfunc(s):
881 self._load()
920 self._load()
882 for d in self._dirs:
921 # OPT: it'd be nice to not load everything here. Unfortunately
883 s._dirs[d] = self._dirs[d].copy()
922 # this makes a mess of the "dirty" state tracking if we don't.
923 self._loadalllazy()
924 sdirs = s._dirs
925 for d, v in self._dirs.iteritems():
926 sdirs[d] = v.copy()
884 s._files = dict.copy(self._files)
927 s._files = dict.copy(self._files)
885 s._flags = dict.copy(self._flags)
928 s._flags = dict.copy(self._flags)
886 if self._loadfunc is _noop:
929 if self._loadfunc is _noop:
@@ -904,6 +947,8 b' class treemanifest(object):'
904 return
947 return
905 t1._load()
948 t1._load()
906 t2._load()
949 t2._load()
950 t1._loadalllazy()
951 t2._loadalllazy()
907 for d, m1 in t1._dirs.iteritems():
952 for d, m1 in t1._dirs.iteritems():
908 if d in t2._dirs:
953 if d in t2._dirs:
909 m2 = t2._dirs[d]
954 m2 = t2._dirs[d]
@@ -929,10 +974,13 b' class treemanifest(object):'
929 self._load()
974 self._load()
930 topdir, subdir = _splittopdir(dir)
975 topdir, subdir = _splittopdir(dir)
931 if topdir:
976 if topdir:
977 if topdir in self._lazydirs:
978 self._loadlazy(topdir)
932 if topdir in self._dirs:
979 if topdir in self._dirs:
933 return self._dirs[topdir].hasdir(subdir)
980 return self._dirs[topdir].hasdir(subdir)
934 return False
981 return False
935 return (dir + '/') in self._dirs
982 dirslash = dir + '/'
983 return dirslash in self._dirs or dirslash in self._lazydirs
936
984
937 def walk(self, match):
985 def walk(self, match):
938 '''Generates matching file names.
986 '''Generates matching file names.
@@ -970,6 +1018,7 b' class treemanifest(object):'
970
1018
971 # yield this dir's files and walk its submanifests
1019 # yield this dir's files and walk its submanifests
972 self._load()
1020 self._load()
1021 self._loadalllazy()
973 for p in sorted(list(self._dirs) + list(self._files)):
1022 for p in sorted(list(self._dirs) + list(self._files)):
974 if p in self._files:
1023 if p in self._files:
975 fullp = self._subpath(p)
1024 fullp = self._subpath(p)
@@ -1006,6 +1055,8 b' class treemanifest(object):'
1006 if fn in self._flags:
1055 if fn in self._flags:
1007 ret._flags[fn] = self._flags[fn]
1056 ret._flags[fn] = self._flags[fn]
1008
1057
1058 # OPT: use visitchildrenset to avoid loading everything
1059 self._loadalllazy()
1009 for dir, subm in self._dirs.iteritems():
1060 for dir, subm in self._dirs.iteritems():
1010 m = subm._matches(match)
1061 m = subm._matches(match)
1011 if not m._isempty():
1062 if not m._isempty():
@@ -1041,6 +1092,9 b' class treemanifest(object):'
1041 return
1092 return
1042 t1._load()
1093 t1._load()
1043 t2._load()
1094 t2._load()
1095 # OPT: do we need to load everything?
1096 t1._loadalllazy()
1097 t2._loadalllazy()
1044 for d, m1 in t1._dirs.iteritems():
1098 for d, m1 in t1._dirs.iteritems():
1045 m2 = t2._dirs.get(d, emptytree)
1099 m2 = t2._dirs.get(d, emptytree)
1046 _diff(m1, m2)
1100 _diff(m1, m2)
@@ -1070,10 +1124,12 b' class treemanifest(object):'
1070 return not self._dirty and not m2._dirty and self._node == m2._node
1124 return not self._dirty and not m2._dirty and self._node == m2._node
1071
1125
1072 def parse(self, text, readsubtree):
1126 def parse(self, text, readsubtree):
1127 selflazy = self._lazydirs
1128 subpath = self._subpath
1073 for f, n, fl in _parse(text):
1129 for f, n, fl in _parse(text):
1074 if fl == 't':
1130 if fl == 't':
1075 f = f + '/'
1131 f = f + '/'
1076 self._dirs[f] = readsubtree(self._subpath(f), n)
1132 selflazy[f] = (subpath(f), n, readsubtree)
1077 elif '/' in f:
1133 elif '/' in f:
1078 # This is a flat manifest, so use __setitem__ and setflag rather
1134 # This is a flat manifest, so use __setitem__ and setflag rather
1079 # than assigning directly to _files and _flags, so we can
1135 # than assigning directly to _files and _flags, so we can
@@ -1100,9 +1156,11 b' class treemanifest(object):'
1100 """
1156 """
1101 self._load()
1157 self._load()
1102 flags = self.flags
1158 flags = self.flags
1159 lazydirs = [(d[:-1], node, 't') for
1160 d, (path, node, readsubtree) in self._lazydirs.iteritems()]
1103 dirs = [(d[:-1], self._dirs[d]._node, 't') for d in self._dirs]
1161 dirs = [(d[:-1], self._dirs[d]._node, 't') for d in self._dirs]
1104 files = [(f, self._files[f], flags(f)) for f in self._files]
1162 files = [(f, self._files[f], flags(f)) for f in self._files]
1105 return _text(sorted(dirs + files))
1163 return _text(sorted(dirs + files + lazydirs))
1106
1164
1107 def read(self, gettext, readsubtree):
1165 def read(self, gettext, readsubtree):
1108 def _load_for_read(s):
1166 def _load_for_read(s):
@@ -1115,6 +1173,11 b' class treemanifest(object):'
1115 m1._load()
1173 m1._load()
1116 m2._load()
1174 m2._load()
1117 emptytree = treemanifest()
1175 emptytree = treemanifest()
1176 # OPT: Do we really need to load everything? Presumably things in lazy
1177 # aren't dirty and don't need to be written.
1178 self._loadalllazy()
1179 m1._loadalllazy()
1180 m2._loadalllazy()
1118 for d, subm in self._dirs.iteritems():
1181 for d, subm in self._dirs.iteritems():
1119 subp1 = m1._dirs.get(d, emptytree)._node
1182 subp1 = m1._dirs.get(d, emptytree)._node
1120 subp2 = m2._dirs.get(d, emptytree)._node
1183 subp2 = m2._dirs.get(d, emptytree)._node
@@ -1134,6 +1197,8 b' class treemanifest(object):'
1134 yield self
1197 yield self
1135
1198
1136 self._load()
1199 self._load()
1200 # OPT: use visitchildrenset to avoid loading everything.
1201 self._loadalllazy()
1137 for d, subm in self._dirs.iteritems():
1202 for d, subm in self._dirs.iteritems():
1138 for subtree in subm.walksubtrees(matcher=matcher):
1203 for subtree in subm.walksubtrees(matcher=matcher):
1139 yield subtree
1204 yield subtree
General Comments 0
You need to be logged in to leave comments. Login now