Show More
@@ -0,0 +1,129 b'' | |||||
|
1 | # pull_logger.py - Logs pulls to a JSON-line file in the repo's VFS. | |||
|
2 | # | |||
|
3 | # Copyright 2022 Pacien TRAN-GIRARD <pacien.trangirard@pacien.net> | |||
|
4 | # | |||
|
5 | # This software may be used and distributed according to the terms of the | |||
|
6 | # GNU General Public License version 2 or any later version. | |||
|
7 | ||||
|
8 | ||||
|
9 | '''logs pull parameters to a file | |||
|
10 | ||||
|
11 | This extension logs the pull parameters, i.e. the remote and common heads, | |||
|
12 | when pulling from the local repository. | |||
|
13 | ||||
|
14 | The collected data should give an idea of the state of a pair of repositories | |||
|
15 | and allow replaying past synchronisations between them. This is particularly | |||
|
16 | useful for working on data exchange, bundling and caching-related | |||
|
17 | optimisations. | |||
|
18 | ||||
|
19 | The record is a JSON-line file located in the repository's VFS at | |||
|
20 | .hg/pull_log.jsonl. | |||
|
21 | ||||
|
22 | Log write failures are not considered fatal: log writes may be skipped for any | |||
|
23 | reason such as insufficient storage or a timeout. | |||
|
24 | ||||
|
25 | The timeouts of the exclusive lock used when writing to the lock file can be | |||
|
26 | configured through the 'timeout.lock' and 'timeout.warn' options of this | |||
|
27 | plugin. Those are not expected to be held for a significant time in practice.:: | |||
|
28 | ||||
|
29 | [pull-logger] | |||
|
30 | timeout.lock = 300 | |||
|
31 | timeout.warn = 100 | |||
|
32 | ||||
|
33 | Note: there is no automatic log rotation and the size of the log is not capped. | |||
|
34 | ''' | |||
|
35 | ||||
|
36 | ||||
|
37 | import json | |||
|
38 | import time | |||
|
39 | ||||
|
40 | from mercurial.i18n import _ | |||
|
41 | from mercurial.utils import stringutil | |||
|
42 | from mercurial import ( | |||
|
43 | error, | |||
|
44 | extensions, | |||
|
45 | lock, | |||
|
46 | registrar, | |||
|
47 | wireprotov1server, | |||
|
48 | ) | |||
|
49 | ||||
|
50 | EXT_NAME = b'pull-logger' | |||
|
51 | EXT_VERSION_CODE = 0 | |||
|
52 | ||||
|
53 | LOG_FILE = b'pull_log.jsonl' | |||
|
54 | LOCK_NAME = LOG_FILE + b'.lock' | |||
|
55 | ||||
|
56 | configtable = {} | |||
|
57 | configitem = registrar.configitem(configtable) | |||
|
58 | configitem(EXT_NAME, b'timeout.lock', default=600) | |||
|
59 | configitem(EXT_NAME, b'timeout.warn', default=120) | |||
|
60 | ||||
|
61 | ||||
|
62 | def wrap_getbundle(orig, repo, proto, others, *args, **kwargs): | |||
|
63 | heads, common = extract_pull_heads(others) | |||
|
64 | log_entry = { | |||
|
65 | 'timestamp': time.time(), | |||
|
66 | 'logger_version': EXT_VERSION_CODE, | |||
|
67 | 'heads': sorted(heads), | |||
|
68 | 'common': sorted(common), | |||
|
69 | } | |||
|
70 | ||||
|
71 | try: | |||
|
72 | write_to_log(repo, log_entry) | |||
|
73 | except (IOError, error.LockError) as err: | |||
|
74 | msg = stringutil.forcebytestr(err) | |||
|
75 | repo.ui.warn(_(b'unable to append to pull log: %s\n') % msg) | |||
|
76 | ||||
|
77 | return orig(repo, proto, others, *args, **kwargs) | |||
|
78 | ||||
|
79 | ||||
|
80 | def extract_pull_heads(bundle_args): | |||
|
81 | opts = wireprotov1server.options( | |||
|
82 | b'getbundle', | |||
|
83 | wireprotov1server.wireprototypes.GETBUNDLE_ARGUMENTS.keys(), | |||
|
84 | bundle_args.copy(), # this call consumes the args destructively | |||
|
85 | ) | |||
|
86 | ||||
|
87 | heads = opts.get(b'heads', b'').decode('utf-8').split(' ') | |||
|
88 | common = opts.get(b'common', b'').decode('utf-8').split(' ') | |||
|
89 | return (heads, common) | |||
|
90 | ||||
|
91 | ||||
|
92 | def write_to_log(repo, entry): | |||
|
93 | locktimeout = repo.ui.configint(EXT_NAME, b'timeout.lock') | |||
|
94 | lockwarntimeout = repo.ui.configint(EXT_NAME, b'timeout.warn') | |||
|
95 | ||||
|
96 | with lock.trylock( | |||
|
97 | ui=repo.ui, | |||
|
98 | vfs=repo.vfs, | |||
|
99 | lockname=LOCK_NAME, | |||
|
100 | timeout=locktimeout, | |||
|
101 | warntimeout=lockwarntimeout, | |||
|
102 | ): | |||
|
103 | with repo.vfs.open(LOG_FILE, b'a+') as logfile: | |||
|
104 | serialised = json.dumps(entry, sort_keys=True) | |||
|
105 | logfile.write(serialised.encode('utf-8')) | |||
|
106 | logfile.write(b'\n') | |||
|
107 | logfile.flush() | |||
|
108 | ||||
|
109 | ||||
|
110 | def reposetup(ui, repo): | |||
|
111 | if repo.local(): | |||
|
112 | repo._wlockfreeprefix.add(LOG_FILE) | |||
|
113 | ||||
|
114 | ||||
|
115 | def uisetup(ui): | |||
|
116 | del wireprotov1server.commands[b'getbundle'] | |||
|
117 | decorator = wireprotov1server.wireprotocommand( | |||
|
118 | name=b'getbundle', | |||
|
119 | args=b'*', | |||
|
120 | permission=b'pull', | |||
|
121 | ) | |||
|
122 | ||||
|
123 | extensions.wrapfunction( | |||
|
124 | container=wireprotov1server, | |||
|
125 | funcname='getbundle', | |||
|
126 | wrapper=wrap_getbundle, | |||
|
127 | ) | |||
|
128 | ||||
|
129 | decorator(wireprotov1server.getbundle) |
@@ -0,0 +1,61 b'' | |||||
|
1 | Check that the pull logger plugins logs pulls | |||
|
2 | ============================================= | |||
|
3 | ||||
|
4 | Enable the extension | |||
|
5 | ||||
|
6 | $ echo "[extensions]" >> $HGRCPATH | |||
|
7 | $ echo "pull-logger = $TESTDIR/../contrib/pull_logger.py" >> $HGRCPATH | |||
|
8 | ||||
|
9 | ||||
|
10 | Check the format of the generated log entries, with a bunch of elements in the | |||
|
11 | common and heads set | |||
|
12 | ||||
|
13 | $ hg init server | |||
|
14 | $ hg -R server debugbuilddag '.*2+2' | |||
|
15 | $ hg clone ssh://user@dummy/server client --rev 0 | |||
|
16 | adding changesets | |||
|
17 | adding manifests | |||
|
18 | adding file changes | |||
|
19 | added 1 changesets with 0 changes to 0 files | |||
|
20 | new changesets 1ea73414a91b | |||
|
21 | updating to branch default | |||
|
22 | 0 files updated, 0 files merged, 0 files removed, 0 files unresolved | |||
|
23 | $ tail -1 server/.hg/pull_log.jsonl | |||
|
24 | {"common": ["0000000000000000000000000000000000000000"], "heads": ["1ea73414a91b0920940797d8fc6a11e447f8ea1e"], "logger_version": 0, "timestamp": *} (glob) | |||
|
25 | $ hg -R client pull --rev 1 --rev 2 | |||
|
26 | pulling from ssh://user@dummy/server | |||
|
27 | searching for changes | |||
|
28 | adding changesets | |||
|
29 | adding manifests | |||
|
30 | adding file changes | |||
|
31 | added 2 changesets with 0 changes to 0 files (+1 heads) | |||
|
32 | new changesets d8736c3a2c84:fa28e81e283b | |||
|
33 | (run 'hg heads' to see heads, 'hg merge' to merge) | |||
|
34 | $ tail -1 server/.hg/pull_log.jsonl | |||
|
35 | {"common": ["1ea73414a91b0920940797d8fc6a11e447f8ea1e"], "heads": ["d8736c3a2c84ee759a2821385804bcb67f266ade", "fa28e81e283b3416de4d48ee0dd2d446e9e38d7c"], "logger_version": 0, "timestamp": *} (glob) | |||
|
36 | $ hg -R client pull --rev 2 --rev 3 | |||
|
37 | pulling from ssh://user@dummy/server | |||
|
38 | searching for changes | |||
|
39 | adding changesets | |||
|
40 | adding manifests | |||
|
41 | adding file changes | |||
|
42 | added 1 changesets with 0 changes to 0 files | |||
|
43 | new changesets 944641ddcaef | |||
|
44 | (run 'hg update' to get a working copy) | |||
|
45 | $ tail -1 server/.hg/pull_log.jsonl | |||
|
46 | {"common": ["1ea73414a91b0920940797d8fc6a11e447f8ea1e", "fa28e81e283b3416de4d48ee0dd2d446e9e38d7c"], "heads": ["944641ddcaef174df7ce1bc2751a5f165129778b", "fa28e81e283b3416de4d48ee0dd2d446e9e38d7c"], "logger_version": 0, "timestamp": *} (glob) | |||
|
47 | ||||
|
48 | ||||
|
49 | Check the number of entries generated in the log when pulling from multiple | |||
|
50 | clients at the same time | |||
|
51 | ||||
|
52 | $ rm -f server/.hg/pull_log.jsonl | |||
|
53 | $ for i in $($TESTDIR/seq.py 32); do | |||
|
54 | > hg clone ssh://user@dummy/server client_$i --rev 0 | |||
|
55 | > done > /dev/null | |||
|
56 | $ for i in $($TESTDIR/seq.py 32); do | |||
|
57 | > hg -R client_$i pull --rev 1 & | |||
|
58 | > done > /dev/null | |||
|
59 | $ wait | |||
|
60 | $ wc -l server/.hg/pull_log.jsonl | |||
|
61 | \s*64 .* (re) |
General Comments 0
You need to be logged in to leave comments.
Login now