##// END OF EJS Templates
contrib: add pull_logger extension...
pacien -
r50403:79105036 default
parent child Browse files
Show More
@@ -0,0 +1,129 b''
1 # pull_logger.py - Logs pulls to a JSON-line file in the repo's VFS.
2 #
3 # Copyright 2022 Pacien TRAN-GIRARD <pacien.trangirard@pacien.net>
4 #
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
7
8
9 '''logs pull parameters to a file
10
11 This extension logs the pull parameters, i.e. the remote and common heads,
12 when pulling from the local repository.
13
14 The collected data should give an idea of the state of a pair of repositories
15 and allow replaying past synchronisations between them. This is particularly
16 useful for working on data exchange, bundling and caching-related
17 optimisations.
18
19 The record is a JSON-line file located in the repository's VFS at
20 .hg/pull_log.jsonl.
21
22 Log write failures are not considered fatal: log writes may be skipped for any
23 reason such as insufficient storage or a timeout.
24
25 The timeouts of the exclusive lock used when writing to the lock file can be
26 configured through the 'timeout.lock' and 'timeout.warn' options of this
27 plugin. Those are not expected to be held for a significant time in practice.::
28
29 [pull-logger]
30 timeout.lock = 300
31 timeout.warn = 100
32
33 Note: there is no automatic log rotation and the size of the log is not capped.
34 '''
35
36
37 import json
38 import time
39
40 from mercurial.i18n import _
41 from mercurial.utils import stringutil
42 from mercurial import (
43 error,
44 extensions,
45 lock,
46 registrar,
47 wireprotov1server,
48 )
49
50 EXT_NAME = b'pull-logger'
51 EXT_VERSION_CODE = 0
52
53 LOG_FILE = b'pull_log.jsonl'
54 LOCK_NAME = LOG_FILE + b'.lock'
55
56 configtable = {}
57 configitem = registrar.configitem(configtable)
58 configitem(EXT_NAME, b'timeout.lock', default=600)
59 configitem(EXT_NAME, b'timeout.warn', default=120)
60
61
62 def wrap_getbundle(orig, repo, proto, others, *args, **kwargs):
63 heads, common = extract_pull_heads(others)
64 log_entry = {
65 'timestamp': time.time(),
66 'logger_version': EXT_VERSION_CODE,
67 'heads': sorted(heads),
68 'common': sorted(common),
69 }
70
71 try:
72 write_to_log(repo, log_entry)
73 except (IOError, error.LockError) as err:
74 msg = stringutil.forcebytestr(err)
75 repo.ui.warn(_(b'unable to append to pull log: %s\n') % msg)
76
77 return orig(repo, proto, others, *args, **kwargs)
78
79
80 def extract_pull_heads(bundle_args):
81 opts = wireprotov1server.options(
82 b'getbundle',
83 wireprotov1server.wireprototypes.GETBUNDLE_ARGUMENTS.keys(),
84 bundle_args.copy(), # this call consumes the args destructively
85 )
86
87 heads = opts.get(b'heads', b'').decode('utf-8').split(' ')
88 common = opts.get(b'common', b'').decode('utf-8').split(' ')
89 return (heads, common)
90
91
92 def write_to_log(repo, entry):
93 locktimeout = repo.ui.configint(EXT_NAME, b'timeout.lock')
94 lockwarntimeout = repo.ui.configint(EXT_NAME, b'timeout.warn')
95
96 with lock.trylock(
97 ui=repo.ui,
98 vfs=repo.vfs,
99 lockname=LOCK_NAME,
100 timeout=locktimeout,
101 warntimeout=lockwarntimeout,
102 ):
103 with repo.vfs.open(LOG_FILE, b'a+') as logfile:
104 serialised = json.dumps(entry, sort_keys=True)
105 logfile.write(serialised.encode('utf-8'))
106 logfile.write(b'\n')
107 logfile.flush()
108
109
110 def reposetup(ui, repo):
111 if repo.local():
112 repo._wlockfreeprefix.add(LOG_FILE)
113
114
115 def uisetup(ui):
116 del wireprotov1server.commands[b'getbundle']
117 decorator = wireprotov1server.wireprotocommand(
118 name=b'getbundle',
119 args=b'*',
120 permission=b'pull',
121 )
122
123 extensions.wrapfunction(
124 container=wireprotov1server,
125 funcname='getbundle',
126 wrapper=wrap_getbundle,
127 )
128
129 decorator(wireprotov1server.getbundle)
@@ -0,0 +1,61 b''
1 Check that the pull logger plugins logs pulls
2 =============================================
3
4 Enable the extension
5
6 $ echo "[extensions]" >> $HGRCPATH
7 $ echo "pull-logger = $TESTDIR/../contrib/pull_logger.py" >> $HGRCPATH
8
9
10 Check the format of the generated log entries, with a bunch of elements in the
11 common and heads set
12
13 $ hg init server
14 $ hg -R server debugbuilddag '.*2+2'
15 $ hg clone ssh://user@dummy/server client --rev 0
16 adding changesets
17 adding manifests
18 adding file changes
19 added 1 changesets with 0 changes to 0 files
20 new changesets 1ea73414a91b
21 updating to branch default
22 0 files updated, 0 files merged, 0 files removed, 0 files unresolved
23 $ tail -1 server/.hg/pull_log.jsonl
24 {"common": ["0000000000000000000000000000000000000000"], "heads": ["1ea73414a91b0920940797d8fc6a11e447f8ea1e"], "logger_version": 0, "timestamp": *} (glob)
25 $ hg -R client pull --rev 1 --rev 2
26 pulling from ssh://user@dummy/server
27 searching for changes
28 adding changesets
29 adding manifests
30 adding file changes
31 added 2 changesets with 0 changes to 0 files (+1 heads)
32 new changesets d8736c3a2c84:fa28e81e283b
33 (run 'hg heads' to see heads, 'hg merge' to merge)
34 $ tail -1 server/.hg/pull_log.jsonl
35 {"common": ["1ea73414a91b0920940797d8fc6a11e447f8ea1e"], "heads": ["d8736c3a2c84ee759a2821385804bcb67f266ade", "fa28e81e283b3416de4d48ee0dd2d446e9e38d7c"], "logger_version": 0, "timestamp": *} (glob)
36 $ hg -R client pull --rev 2 --rev 3
37 pulling from ssh://user@dummy/server
38 searching for changes
39 adding changesets
40 adding manifests
41 adding file changes
42 added 1 changesets with 0 changes to 0 files
43 new changesets 944641ddcaef
44 (run 'hg update' to get a working copy)
45 $ tail -1 server/.hg/pull_log.jsonl
46 {"common": ["1ea73414a91b0920940797d8fc6a11e447f8ea1e", "fa28e81e283b3416de4d48ee0dd2d446e9e38d7c"], "heads": ["944641ddcaef174df7ce1bc2751a5f165129778b", "fa28e81e283b3416de4d48ee0dd2d446e9e38d7c"], "logger_version": 0, "timestamp": *} (glob)
47
48
49 Check the number of entries generated in the log when pulling from multiple
50 clients at the same time
51
52 $ rm -f server/.hg/pull_log.jsonl
53 $ for i in $($TESTDIR/seq.py 32); do
54 > hg clone ssh://user@dummy/server client_$i --rev 0
55 > done > /dev/null
56 $ for i in $($TESTDIR/seq.py 32); do
57 > hg -R client_$i pull --rev 1 &
58 > done > /dev/null
59 $ wait
60 $ wc -l server/.hg/pull_log.jsonl
61 \s*64 .* (re)
General Comments 0
You need to be logged in to leave comments. Login now