##// END OF EJS Templates
generate-churning-bundle: fix script for python3...
marmoute -
r52170:d7155949 stable
parent child Browse files
Show More
@@ -1,142 +1,142 b''
1 #!/usr/bin/env python3
1 #!/usr/bin/env python3
2 #
2 #
3 # generate-branchy-bundle - generate a branch for a "large" branchy repository
3 # generate-branchy-bundle - generate a branch for a "large" branchy repository
4 #
4 #
5 # Copyright 2018 Octobus, contact@octobus.net
5 # Copyright 2018 Octobus, contact@octobus.net
6 #
6 #
7 # This software may be used and distributed according to the terms of the
7 # This software may be used and distributed according to the terms of the
8 # GNU General Public License version 2 or any later version.
8 # GNU General Public License version 2 or any later version.
9 #
9 #
10 # This script generates a repository suitable for testing delta computation
10 # This script generates a repository suitable for testing delta computation
11 # strategies.
11 # strategies.
12 #
12 #
13 # The repository update a single "large" file with many updates. One fixed part
13 # The repository update a single "large" file with many updates. One fixed part
14 # of the files always get updated while the rest of the lines get updated over
14 # of the files always get updated while the rest of the lines get updated over
15 # time. This update happens over many topological branches, some getting merged
15 # time. This update happens over many topological branches, some getting merged
16 # back.
16 # back.
17 #
17 #
18 # Running with `chg` in your path and `CHGHG` set is recommended for speed.
18 # Running with `chg` in your path and `CHGHG` set is recommended for speed.
19
19
20
20
21 import hashlib
21 import hashlib
22 import os
22 import os
23 import shutil
23 import shutil
24 import subprocess
24 import subprocess
25 import sys
25 import sys
26 import tempfile
26 import tempfile
27
27
28 BUNDLE_NAME = 'big-file-churn.hg'
28 BUNDLE_NAME = 'big-file-churn.hg'
29
29
30 # constants for generating the repository
30 # constants for generating the repository
31 NB_CHANGESET = 5000
31 NB_CHANGESET = 5000
32 PERIOD_MERGING = 8
32 PERIOD_MERGING = 8
33 PERIOD_BRANCHING = 7
33 PERIOD_BRANCHING = 7
34 MOVE_BACK_MIN = 3
34 MOVE_BACK_MIN = 3
35 MOVE_BACK_RANGE = 5
35 MOVE_BACK_RANGE = 5
36
36
37 # constants for generating the large file we keep updating
37 # constants for generating the large file we keep updating
38 #
38 #
39 # At each revision, the beginning on the file change,
39 # At each revision, the beginning on the file change,
40 # and set of other lines changes too.
40 # and set of other lines changes too.
41 FILENAME = 'SPARSE-REVLOG-TEST-FILE'
41 FILENAME = 'SPARSE-REVLOG-TEST-FILE'
42 NB_LINES = 10500
42 NB_LINES = 10500
43 ALWAYS_CHANGE_LINES = 500
43 ALWAYS_CHANGE_LINES = 500
44 OTHER_CHANGES = 300
44 OTHER_CHANGES = 300
45
45
46
46
47 def nextcontent(previous_content):
47 def nextcontent(previous_content):
48 """utility to produce a new file content from the previous one"""
48 """utility to produce a new file content from the previous one"""
49 return hashlib.md5(previous_content).hexdigest()
49 return hashlib.md5(previous_content).hexdigest().encode('ascii')
50
50
51
51
52 def filecontent(iteridx, oldcontent):
52 def filecontent(iteridx, oldcontent):
53 """generate a new file content
53 """generate a new file content
54
54
55 The content is generated according the iteration index and previous
55 The content is generated according the iteration index and previous
56 content"""
56 content"""
57
57
58 # initial call
58 # initial call
59 if iteridx is None:
59 if iteridx is None:
60 current = ''
60 current = b''
61 else:
61 else:
62 current = str(iteridx)
62 current = b"%d" % iteridx
63
63
64 for idx in range(NB_LINES):
64 for idx in range(NB_LINES):
65 do_change_line = True
65 do_change_line = True
66 if oldcontent is not None and ALWAYS_CHANGE_LINES < idx:
66 if oldcontent is not None and ALWAYS_CHANGE_LINES < idx:
67 do_change_line = not ((idx - iteridx) % OTHER_CHANGES)
67 do_change_line = not ((idx - iteridx) % OTHER_CHANGES)
68
68
69 if do_change_line:
69 if do_change_line:
70 to_write = current + '\n'
70 to_write = current + b'\n'
71 current = nextcontent(current)
71 current = nextcontent(current)
72 else:
72 else:
73 to_write = oldcontent[idx]
73 to_write = oldcontent[idx]
74 yield to_write
74 yield to_write
75
75
76
76
77 def updatefile(filename, idx):
77 def updatefile(filename, idx):
78 """update <filename> to be at appropriate content for iteration <idx>"""
78 """update <filename> to be at appropriate content for iteration <idx>"""
79 existing = None
79 existing = None
80 if idx is not None:
80 if idx is not None:
81 with open(filename, 'rb') as old:
81 with open(filename, 'rb') as old:
82 existing = old.readlines()
82 existing = old.readlines()
83 with open(filename, 'wb') as target:
83 with open(filename, 'wb') as target:
84 for line in filecontent(idx, existing):
84 for line in filecontent(idx, existing):
85 target.write(line)
85 target.write(line)
86
86
87
87
88 def hg(command, *args):
88 def hg(command, *args):
89 """call a mercurial command with appropriate config and argument"""
89 """call a mercurial command with appropriate config and argument"""
90 env = os.environ.copy()
90 env = os.environ.copy()
91 if 'CHGHG' in env:
91 if 'CHGHG' in env:
92 full_cmd = ['chg']
92 full_cmd = ['chg']
93 else:
93 else:
94 full_cmd = ['hg']
94 full_cmd = ['hg']
95 full_cmd.append('--quiet')
95 full_cmd.append('--quiet')
96 full_cmd.append(command)
96 full_cmd.append(command)
97 if command == 'commit':
97 if command == 'commit':
98 # reproducible commit metadata
98 # reproducible commit metadata
99 full_cmd.extend(['--date', '0 0', '--user', 'test'])
99 full_cmd.extend(['--date', '0 0', '--user', 'test'])
100 elif command == 'merge':
100 elif command == 'merge':
101 # avoid conflicts by picking the local variant
101 # avoid conflicts by picking the local variant
102 full_cmd.extend(['--tool', ':merge-local'])
102 full_cmd.extend(['--tool', ':merge-local'])
103 full_cmd.extend(args)
103 full_cmd.extend(args)
104 env['HGRCPATH'] = ''
104 env['HGRCPATH'] = ''
105 return subprocess.check_call(full_cmd, env=env)
105 return subprocess.check_call(full_cmd, env=env)
106
106
107
107
108 def run(target):
108 def run(target):
109 tmpdir = tempfile.mkdtemp(prefix='tmp-hg-test-big-file-bundle-')
109 tmpdir = tempfile.mkdtemp(prefix='tmp-hg-test-big-file-bundle-')
110 try:
110 try:
111 os.chdir(tmpdir)
111 os.chdir(tmpdir)
112 hg('init')
112 hg('init')
113 updatefile(FILENAME, None)
113 updatefile(FILENAME, None)
114 hg('commit', '--addremove', '--message', 'initial commit')
114 hg('commit', '--addremove', '--message', 'initial commit')
115 for idx in range(1, NB_CHANGESET + 1):
115 for idx in range(1, NB_CHANGESET + 1):
116 if sys.stdout.isatty():
116 if sys.stdout.isatty():
117 print("generating commit #%d/%d" % (idx, NB_CHANGESET))
117 print("generating commit #%d/%d" % (idx, NB_CHANGESET))
118 if (idx % PERIOD_BRANCHING) == 0:
118 if (idx % PERIOD_BRANCHING) == 0:
119 move_back = MOVE_BACK_MIN + (idx % MOVE_BACK_RANGE)
119 move_back = MOVE_BACK_MIN + (idx % MOVE_BACK_RANGE)
120 hg('update', ".~%d" % move_back)
120 hg('update', ".~%d" % move_back)
121 if (idx % PERIOD_MERGING) == 0:
121 if (idx % PERIOD_MERGING) == 0:
122 hg('merge', 'min(head())')
122 hg('merge', 'min(head())')
123 updatefile(FILENAME, idx)
123 updatefile(FILENAME, idx)
124 hg('commit', '--message', 'commit #%d' % idx)
124 hg('commit', '--message', 'commit #%d' % idx)
125 hg('bundle', '--all', target, '--config', 'devel.bundle.delta=p1')
125 hg('bundle', '--all', target, '--config', 'devel.bundle.delta=p1')
126 with open(target, 'rb') as bundle:
126 with open(target, 'rb') as bundle:
127 data = bundle.read()
127 data = bundle.read()
128 digest = hashlib.md5(data).hexdigest()
128 digest = hashlib.md5(data).hexdigest()
129 with open(target + '.md5', 'wb') as md5file:
129 with open(target + '.md5', 'wb') as md5file:
130 md5file.write(digest + '\n')
130 md5file.write(digest.encode('ascii') + b'\n')
131 if sys.stdout.isatty():
131 if sys.stdout.isatty():
132 print('bundle generated at "%s" md5: %s' % (target, digest))
132 print('bundle generated at "%s" md5: %s' % (target, digest))
133
133
134 finally:
134 finally:
135 shutil.rmtree(tmpdir)
135 shutil.rmtree(tmpdir)
136 return 0
136 return 0
137
137
138
138
139 if __name__ == '__main__':
139 if __name__ == '__main__':
140 orig = os.path.realpath(os.path.dirname(sys.argv[0]))
140 orig = os.path.realpath(os.path.dirname(sys.argv[0]))
141 target = os.path.join(orig, os.pardir, 'cache', BUNDLE_NAME)
141 target = os.path.join(orig, os.pardir, 'cache', BUNDLE_NAME)
142 sys.exit(run(target))
142 sys.exit(run(target))
General Comments 0
You need to be logged in to leave comments. Login now