Show More
@@ -1,142 +1,142 b'' | |||||
1 | #!/usr/bin/env python3 |
|
1 | #!/usr/bin/env python3 | |
2 | # |
|
2 | # | |
3 | # generate-branchy-bundle - generate a branch for a "large" branchy repository |
|
3 | # generate-branchy-bundle - generate a branch for a "large" branchy repository | |
4 | # |
|
4 | # | |
5 | # Copyright 2018 Octobus, contact@octobus.net |
|
5 | # Copyright 2018 Octobus, contact@octobus.net | |
6 | # |
|
6 | # | |
7 | # This software may be used and distributed according to the terms of the |
|
7 | # This software may be used and distributed according to the terms of the | |
8 | # GNU General Public License version 2 or any later version. |
|
8 | # GNU General Public License version 2 or any later version. | |
9 | # |
|
9 | # | |
10 | # This script generates a repository suitable for testing delta computation |
|
10 | # This script generates a repository suitable for testing delta computation | |
11 | # strategies. |
|
11 | # strategies. | |
12 | # |
|
12 | # | |
13 | # The repository update a single "large" file with many updates. One fixed part |
|
13 | # The repository update a single "large" file with many updates. One fixed part | |
14 | # of the files always get updated while the rest of the lines get updated over |
|
14 | # of the files always get updated while the rest of the lines get updated over | |
15 | # time. This update happens over many topological branches, some getting merged |
|
15 | # time. This update happens over many topological branches, some getting merged | |
16 | # back. |
|
16 | # back. | |
17 | # |
|
17 | # | |
18 | # Running with `chg` in your path and `CHGHG` set is recommended for speed. |
|
18 | # Running with `chg` in your path and `CHGHG` set is recommended for speed. | |
19 |
|
19 | |||
20 |
|
20 | |||
21 | import hashlib |
|
21 | import hashlib | |
22 | import os |
|
22 | import os | |
23 | import shutil |
|
23 | import shutil | |
24 | import subprocess |
|
24 | import subprocess | |
25 | import sys |
|
25 | import sys | |
26 | import tempfile |
|
26 | import tempfile | |
27 |
|
27 | |||
28 | BUNDLE_NAME = 'big-file-churn.hg' |
|
28 | BUNDLE_NAME = 'big-file-churn.hg' | |
29 |
|
29 | |||
30 | # constants for generating the repository |
|
30 | # constants for generating the repository | |
31 | NB_CHANGESET = 5000 |
|
31 | NB_CHANGESET = 5000 | |
32 | PERIOD_MERGING = 8 |
|
32 | PERIOD_MERGING = 8 | |
33 | PERIOD_BRANCHING = 7 |
|
33 | PERIOD_BRANCHING = 7 | |
34 | MOVE_BACK_MIN = 3 |
|
34 | MOVE_BACK_MIN = 3 | |
35 | MOVE_BACK_RANGE = 5 |
|
35 | MOVE_BACK_RANGE = 5 | |
36 |
|
36 | |||
37 | # constants for generating the large file we keep updating |
|
37 | # constants for generating the large file we keep updating | |
38 | # |
|
38 | # | |
39 | # At each revision, the beginning on the file change, |
|
39 | # At each revision, the beginning on the file change, | |
40 | # and set of other lines changes too. |
|
40 | # and set of other lines changes too. | |
41 | FILENAME = 'SPARSE-REVLOG-TEST-FILE' |
|
41 | FILENAME = 'SPARSE-REVLOG-TEST-FILE' | |
42 | NB_LINES = 10500 |
|
42 | NB_LINES = 10500 | |
43 | ALWAYS_CHANGE_LINES = 500 |
|
43 | ALWAYS_CHANGE_LINES = 500 | |
44 | OTHER_CHANGES = 300 |
|
44 | OTHER_CHANGES = 300 | |
45 |
|
45 | |||
46 |
|
46 | |||
47 | def nextcontent(previous_content): |
|
47 | def nextcontent(previous_content): | |
48 | """utility to produce a new file content from the previous one""" |
|
48 | """utility to produce a new file content from the previous one""" | |
49 | return hashlib.md5(previous_content).hexdigest() |
|
49 | return hashlib.md5(previous_content).hexdigest().encode('ascii') | |
50 |
|
50 | |||
51 |
|
51 | |||
52 | def filecontent(iteridx, oldcontent): |
|
52 | def filecontent(iteridx, oldcontent): | |
53 | """generate a new file content |
|
53 | """generate a new file content | |
54 |
|
54 | |||
55 | The content is generated according the iteration index and previous |
|
55 | The content is generated according the iteration index and previous | |
56 | content""" |
|
56 | content""" | |
57 |
|
57 | |||
58 | # initial call |
|
58 | # initial call | |
59 | if iteridx is None: |
|
59 | if iteridx is None: | |
60 | current = '' |
|
60 | current = b'' | |
61 | else: |
|
61 | else: | |
62 |
current = |
|
62 | current = b"%d" % iteridx | |
63 |
|
63 | |||
64 | for idx in range(NB_LINES): |
|
64 | for idx in range(NB_LINES): | |
65 | do_change_line = True |
|
65 | do_change_line = True | |
66 | if oldcontent is not None and ALWAYS_CHANGE_LINES < idx: |
|
66 | if oldcontent is not None and ALWAYS_CHANGE_LINES < idx: | |
67 | do_change_line = not ((idx - iteridx) % OTHER_CHANGES) |
|
67 | do_change_line = not ((idx - iteridx) % OTHER_CHANGES) | |
68 |
|
68 | |||
69 | if do_change_line: |
|
69 | if do_change_line: | |
70 | to_write = current + '\n' |
|
70 | to_write = current + b'\n' | |
71 | current = nextcontent(current) |
|
71 | current = nextcontent(current) | |
72 | else: |
|
72 | else: | |
73 | to_write = oldcontent[idx] |
|
73 | to_write = oldcontent[idx] | |
74 | yield to_write |
|
74 | yield to_write | |
75 |
|
75 | |||
76 |
|
76 | |||
77 | def updatefile(filename, idx): |
|
77 | def updatefile(filename, idx): | |
78 | """update <filename> to be at appropriate content for iteration <idx>""" |
|
78 | """update <filename> to be at appropriate content for iteration <idx>""" | |
79 | existing = None |
|
79 | existing = None | |
80 | if idx is not None: |
|
80 | if idx is not None: | |
81 | with open(filename, 'rb') as old: |
|
81 | with open(filename, 'rb') as old: | |
82 | existing = old.readlines() |
|
82 | existing = old.readlines() | |
83 | with open(filename, 'wb') as target: |
|
83 | with open(filename, 'wb') as target: | |
84 | for line in filecontent(idx, existing): |
|
84 | for line in filecontent(idx, existing): | |
85 | target.write(line) |
|
85 | target.write(line) | |
86 |
|
86 | |||
87 |
|
87 | |||
88 | def hg(command, *args): |
|
88 | def hg(command, *args): | |
89 | """call a mercurial command with appropriate config and argument""" |
|
89 | """call a mercurial command with appropriate config and argument""" | |
90 | env = os.environ.copy() |
|
90 | env = os.environ.copy() | |
91 | if 'CHGHG' in env: |
|
91 | if 'CHGHG' in env: | |
92 | full_cmd = ['chg'] |
|
92 | full_cmd = ['chg'] | |
93 | else: |
|
93 | else: | |
94 | full_cmd = ['hg'] |
|
94 | full_cmd = ['hg'] | |
95 | full_cmd.append('--quiet') |
|
95 | full_cmd.append('--quiet') | |
96 | full_cmd.append(command) |
|
96 | full_cmd.append(command) | |
97 | if command == 'commit': |
|
97 | if command == 'commit': | |
98 | # reproducible commit metadata |
|
98 | # reproducible commit metadata | |
99 | full_cmd.extend(['--date', '0 0', '--user', 'test']) |
|
99 | full_cmd.extend(['--date', '0 0', '--user', 'test']) | |
100 | elif command == 'merge': |
|
100 | elif command == 'merge': | |
101 | # avoid conflicts by picking the local variant |
|
101 | # avoid conflicts by picking the local variant | |
102 | full_cmd.extend(['--tool', ':merge-local']) |
|
102 | full_cmd.extend(['--tool', ':merge-local']) | |
103 | full_cmd.extend(args) |
|
103 | full_cmd.extend(args) | |
104 | env['HGRCPATH'] = '' |
|
104 | env['HGRCPATH'] = '' | |
105 | return subprocess.check_call(full_cmd, env=env) |
|
105 | return subprocess.check_call(full_cmd, env=env) | |
106 |
|
106 | |||
107 |
|
107 | |||
108 | def run(target): |
|
108 | def run(target): | |
109 | tmpdir = tempfile.mkdtemp(prefix='tmp-hg-test-big-file-bundle-') |
|
109 | tmpdir = tempfile.mkdtemp(prefix='tmp-hg-test-big-file-bundle-') | |
110 | try: |
|
110 | try: | |
111 | os.chdir(tmpdir) |
|
111 | os.chdir(tmpdir) | |
112 | hg('init') |
|
112 | hg('init') | |
113 | updatefile(FILENAME, None) |
|
113 | updatefile(FILENAME, None) | |
114 | hg('commit', '--addremove', '--message', 'initial commit') |
|
114 | hg('commit', '--addremove', '--message', 'initial commit') | |
115 | for idx in range(1, NB_CHANGESET + 1): |
|
115 | for idx in range(1, NB_CHANGESET + 1): | |
116 | if sys.stdout.isatty(): |
|
116 | if sys.stdout.isatty(): | |
117 | print("generating commit #%d/%d" % (idx, NB_CHANGESET)) |
|
117 | print("generating commit #%d/%d" % (idx, NB_CHANGESET)) | |
118 | if (idx % PERIOD_BRANCHING) == 0: |
|
118 | if (idx % PERIOD_BRANCHING) == 0: | |
119 | move_back = MOVE_BACK_MIN + (idx % MOVE_BACK_RANGE) |
|
119 | move_back = MOVE_BACK_MIN + (idx % MOVE_BACK_RANGE) | |
120 | hg('update', ".~%d" % move_back) |
|
120 | hg('update', ".~%d" % move_back) | |
121 | if (idx % PERIOD_MERGING) == 0: |
|
121 | if (idx % PERIOD_MERGING) == 0: | |
122 | hg('merge', 'min(head())') |
|
122 | hg('merge', 'min(head())') | |
123 | updatefile(FILENAME, idx) |
|
123 | updatefile(FILENAME, idx) | |
124 | hg('commit', '--message', 'commit #%d' % idx) |
|
124 | hg('commit', '--message', 'commit #%d' % idx) | |
125 | hg('bundle', '--all', target, '--config', 'devel.bundle.delta=p1') |
|
125 | hg('bundle', '--all', target, '--config', 'devel.bundle.delta=p1') | |
126 | with open(target, 'rb') as bundle: |
|
126 | with open(target, 'rb') as bundle: | |
127 | data = bundle.read() |
|
127 | data = bundle.read() | |
128 | digest = hashlib.md5(data).hexdigest() |
|
128 | digest = hashlib.md5(data).hexdigest() | |
129 | with open(target + '.md5', 'wb') as md5file: |
|
129 | with open(target + '.md5', 'wb') as md5file: | |
130 | md5file.write(digest + '\n') |
|
130 | md5file.write(digest.encode('ascii') + b'\n') | |
131 | if sys.stdout.isatty(): |
|
131 | if sys.stdout.isatty(): | |
132 | print('bundle generated at "%s" md5: %s' % (target, digest)) |
|
132 | print('bundle generated at "%s" md5: %s' % (target, digest)) | |
133 |
|
133 | |||
134 | finally: |
|
134 | finally: | |
135 | shutil.rmtree(tmpdir) |
|
135 | shutil.rmtree(tmpdir) | |
136 | return 0 |
|
136 | return 0 | |
137 |
|
137 | |||
138 |
|
138 | |||
139 | if __name__ == '__main__': |
|
139 | if __name__ == '__main__': | |
140 | orig = os.path.realpath(os.path.dirname(sys.argv[0])) |
|
140 | orig = os.path.realpath(os.path.dirname(sys.argv[0])) | |
141 | target = os.path.join(orig, os.pardir, 'cache', BUNDLE_NAME) |
|
141 | target = os.path.join(orig, os.pardir, 'cache', BUNDLE_NAME) | |
142 | sys.exit(run(target)) |
|
142 | sys.exit(run(target)) |
General Comments 0
You need to be logged in to leave comments.
Login now