##// END OF EJS Templates
infinitepush: fix `{get,put}_args` formatting on Python 3...
Connor Sheehan -
r45753:e285655c stable
parent child Browse files
Show More
@@ -1,183 +1,198 b''
1 # This software may be used and distributed according to the terms of the
1 # This software may be used and distributed according to the terms of the
2 # GNU General Public License version 2 or any later version.
2 # GNU General Public License version 2 or any later version.
3
3
4 # based on bundleheads extension by Gregory Szorc <gps@mozilla.com>
4 # based on bundleheads extension by Gregory Szorc <gps@mozilla.com>
5
5
6 from __future__ import absolute_import
6 from __future__ import absolute_import
7
7
8 import abc
8 import abc
9 import os
9 import os
10 import subprocess
10 import subprocess
11 import tempfile
11 import tempfile
12
12
13 from mercurial.pycompat import open
13 from mercurial.pycompat import open
14 from mercurial import (
14 from mercurial import (
15 node,
15 node,
16 pycompat,
16 pycompat,
17 )
17 )
18 from mercurial.utils import (
18 from mercurial.utils import (
19 hashutil,
19 hashutil,
20 procutil,
20 procutil,
21 )
21 )
22
22
23
23
24 class BundleWriteException(Exception):
24 class BundleWriteException(Exception):
25 pass
25 pass
26
26
27
27
28 class BundleReadException(Exception):
28 class BundleReadException(Exception):
29 pass
29 pass
30
30
31
31
32 class abstractbundlestore(object): # pytype: disable=ignored-metaclass
32 class abstractbundlestore(object): # pytype: disable=ignored-metaclass
33 """Defines the interface for bundle stores.
33 """Defines the interface for bundle stores.
34
34
35 A bundle store is an entity that stores raw bundle data. It is a simple
35 A bundle store is an entity that stores raw bundle data. It is a simple
36 key-value store. However, the keys are chosen by the store. The keys can
36 key-value store. However, the keys are chosen by the store. The keys can
37 be any Python object understood by the corresponding bundle index (see
37 be any Python object understood by the corresponding bundle index (see
38 ``abstractbundleindex`` below).
38 ``abstractbundleindex`` below).
39 """
39 """
40
40
41 __metaclass__ = abc.ABCMeta
41 __metaclass__ = abc.ABCMeta
42
42
43 @abc.abstractmethod
43 @abc.abstractmethod
44 def write(self, data):
44 def write(self, data):
45 """Write bundle data to the store.
45 """Write bundle data to the store.
46
46
47 This function receives the raw data to be written as a str.
47 This function receives the raw data to be written as a str.
48 Throws BundleWriteException
48 Throws BundleWriteException
49 The key of the written data MUST be returned.
49 The key of the written data MUST be returned.
50 """
50 """
51
51
52 @abc.abstractmethod
52 @abc.abstractmethod
53 def read(self, key):
53 def read(self, key):
54 """Obtain bundle data for a key.
54 """Obtain bundle data for a key.
55
55
56 Returns None if the bundle isn't known.
56 Returns None if the bundle isn't known.
57 Throws BundleReadException
57 Throws BundleReadException
58 The returned object should be a file object supporting read()
58 The returned object should be a file object supporting read()
59 and close().
59 and close().
60 """
60 """
61
61
62
62
63 class filebundlestore(object):
63 class filebundlestore(object):
64 """bundle store in filesystem
64 """bundle store in filesystem
65
65
66 meant for storing bundles somewhere on disk and on network filesystems
66 meant for storing bundles somewhere on disk and on network filesystems
67 """
67 """
68
68
69 def __init__(self, ui, repo):
69 def __init__(self, ui, repo):
70 self.ui = ui
70 self.ui = ui
71 self.repo = repo
71 self.repo = repo
72 self.storepath = ui.configpath(b'scratchbranch', b'storepath')
72 self.storepath = ui.configpath(b'scratchbranch', b'storepath')
73 if not self.storepath:
73 if not self.storepath:
74 self.storepath = self.repo.vfs.join(
74 self.storepath = self.repo.vfs.join(
75 b"scratchbranches", b"filebundlestore"
75 b"scratchbranches", b"filebundlestore"
76 )
76 )
77 if not os.path.exists(self.storepath):
77 if not os.path.exists(self.storepath):
78 os.makedirs(self.storepath)
78 os.makedirs(self.storepath)
79
79
80 def _dirpath(self, hashvalue):
80 def _dirpath(self, hashvalue):
81 """First two bytes of the hash are the name of the upper
81 """First two bytes of the hash are the name of the upper
82 level directory, next two bytes are the name of the
82 level directory, next two bytes are the name of the
83 next level directory"""
83 next level directory"""
84 return os.path.join(self.storepath, hashvalue[0:2], hashvalue[2:4])
84 return os.path.join(self.storepath, hashvalue[0:2], hashvalue[2:4])
85
85
86 def _filepath(self, filename):
86 def _filepath(self, filename):
87 return os.path.join(self._dirpath(filename), filename)
87 return os.path.join(self._dirpath(filename), filename)
88
88
89 def write(self, data):
89 def write(self, data):
90 filename = node.hex(hashutil.sha1(data).digest())
90 filename = node.hex(hashutil.sha1(data).digest())
91 dirpath = self._dirpath(filename)
91 dirpath = self._dirpath(filename)
92
92
93 if not os.path.exists(dirpath):
93 if not os.path.exists(dirpath):
94 os.makedirs(dirpath)
94 os.makedirs(dirpath)
95
95
96 with open(self._filepath(filename), b'wb') as f:
96 with open(self._filepath(filename), b'wb') as f:
97 f.write(data)
97 f.write(data)
98
98
99 return filename
99 return filename
100
100
101 def read(self, key):
101 def read(self, key):
102 try:
102 try:
103 with open(self._filepath(key), b'rb') as f:
103 with open(self._filepath(key), b'rb') as f:
104 return f.read()
104 return f.read()
105 except IOError:
105 except IOError:
106 return None
106 return None
107
107
108
108
109 def format_placeholders_args(args, filename=None, handle=None):
110 """Formats `args` with Infinitepush replacements.
111
112 Hack to get `str.format()`-ed strings working in a BC way with
113 bytes.
114 """
115 formatted_args = []
116 for arg in args:
117 if filename and arg == b'{filename}':
118 formatted_args.append(filename)
119 elif handle and arg == b'{handle}':
120 formatted_args.append(handle)
121 else:
122 formatted_args.append(arg)
123 return formatted_args
124
125
109 class externalbundlestore(abstractbundlestore):
126 class externalbundlestore(abstractbundlestore):
110 def __init__(self, put_binary, put_args, get_binary, get_args):
127 def __init__(self, put_binary, put_args, get_binary, get_args):
111 """
128 """
112 `put_binary` - path to binary file which uploads bundle to external
129 `put_binary` - path to binary file which uploads bundle to external
113 storage and prints key to stdout
130 storage and prints key to stdout
114 `put_args` - format string with additional args to `put_binary`
131 `put_args` - format string with additional args to `put_binary`
115 {filename} replacement field can be used.
132 {filename} replacement field can be used.
116 `get_binary` - path to binary file which accepts filename and key
133 `get_binary` - path to binary file which accepts filename and key
117 (in that order), downloads bundle from store and saves it to file
134 (in that order), downloads bundle from store and saves it to file
118 `get_args` - format string with additional args to `get_binary`.
135 `get_args` - format string with additional args to `get_binary`.
119 {filename} and {handle} replacement field can be used.
136 {filename} and {handle} replacement field can be used.
120 """
137 """
121
138
122 self.put_args = put_args
139 self.put_args = put_args
123 self.get_args = get_args
140 self.get_args = get_args
124 self.put_binary = put_binary
141 self.put_binary = put_binary
125 self.get_binary = get_binary
142 self.get_binary = get_binary
126
143
127 def _call_binary(self, args):
144 def _call_binary(self, args):
128 p = subprocess.Popen(
145 p = subprocess.Popen(
129 pycompat.rapply(procutil.tonativestr, args),
146 pycompat.rapply(procutil.tonativestr, args),
130 stdout=subprocess.PIPE,
147 stdout=subprocess.PIPE,
131 stderr=subprocess.PIPE,
148 stderr=subprocess.PIPE,
132 close_fds=True,
149 close_fds=True,
133 )
150 )
134 stdout, stderr = p.communicate()
151 stdout, stderr = p.communicate()
135 returncode = p.returncode
152 returncode = p.returncode
136 return returncode, stdout, stderr
153 return returncode, stdout, stderr
137
154
138 def write(self, data):
155 def write(self, data):
139 # Won't work on windows because you can't open file second time without
156 # Won't work on windows because you can't open file second time without
140 # closing it
157 # closing it
141 # TODO: rewrite without str.format() and replace NamedTemporaryFile()
158 # TODO: rewrite without str.format() and replace NamedTemporaryFile()
142 # with pycompat.namedtempfile()
159 # with pycompat.namedtempfile()
143 with pycompat.namedtempfile() as temp:
160 with pycompat.namedtempfile() as temp:
144 temp.write(data)
161 temp.write(data)
145 temp.flush()
162 temp.flush()
146 temp.seek(0)
163 temp.seek(0)
147 formatted_args = [
164 formatted_args = format_placeholders_args(
148 arg.format(filename=temp.name) for arg in self.put_args
165 self.put_args, filename=temp.name
149 ]
166 )
150 returncode, stdout, stderr = self._call_binary(
167 returncode, stdout, stderr = self._call_binary(
151 [self.put_binary] + formatted_args
168 [self.put_binary] + formatted_args
152 )
169 )
153
170
154 if returncode != 0:
171 if returncode != 0:
155 raise BundleWriteException(
172 raise BundleWriteException(
156 b'Failed to upload to external store: %s' % stderr
173 b'Failed to upload to external store: %s' % stderr
157 )
174 )
158 stdout_lines = stdout.splitlines()
175 stdout_lines = stdout.splitlines()
159 if len(stdout_lines) == 1:
176 if len(stdout_lines) == 1:
160 return stdout_lines[0]
177 return stdout_lines[0]
161 else:
178 else:
162 raise BundleWriteException(
179 raise BundleWriteException(
163 b'Bad output from %s: %s' % (self.put_binary, stdout)
180 b'Bad output from %s: %s' % (self.put_binary, stdout)
164 )
181 )
165
182
166 def read(self, handle):
183 def read(self, handle):
167 # Won't work on windows because you can't open file second time without
184 # Won't work on windows because you can't open file second time without
168 # closing it
185 # closing it
169 # TODO: rewrite without str.format()
170 with pycompat.namedtempfile() as temp:
186 with pycompat.namedtempfile() as temp:
171 formatted_args = [
187 formatted_args = format_placeholders_args(
172 arg.format(filename=temp.name, handle=handle)
188 self.get_args, filename=temp.name, handle=handle
173 for arg in self.get_args
189 )
174 ]
175 returncode, stdout, stderr = self._call_binary(
190 returncode, stdout, stderr = self._call_binary(
176 [self.get_binary] + formatted_args
191 [self.get_binary] + formatted_args
177 )
192 )
178
193
179 if returncode != 0:
194 if returncode != 0:
180 raise BundleReadException(
195 raise BundleReadException(
181 b'Failed to download from external store: %s' % stderr
196 b'Failed to download from external store: %s' % stderr
182 )
197 )
183 return temp.read()
198 return temp.read()
General Comments 0
You need to be logged in to leave comments. Login now