##// END OF EJS Templates
inifinitepush: fix filebundlestore to close file
Yuya Nishihara -
r37814:968ac00c stable
parent child Browse files
Show More
@@ -1,155 +1,154 b''
1 # This software may be used and distributed according to the terms of the
1 # This software may be used and distributed according to the terms of the
2 # GNU General Public License version 2 or any later version.
2 # GNU General Public License version 2 or any later version.
3
3
4 # based on bundleheads extension by Gregory Szorc <gps@mozilla.com>
4 # based on bundleheads extension by Gregory Szorc <gps@mozilla.com>
5
5
6 from __future__ import absolute_import
6 from __future__ import absolute_import
7
7
8 import abc
8 import abc
9 import hashlib
9 import hashlib
10 import os
10 import os
11 import subprocess
11 import subprocess
12 import tempfile
12 import tempfile
13
13
14 NamedTemporaryFile = tempfile.NamedTemporaryFile
14 NamedTemporaryFile = tempfile.NamedTemporaryFile
15
15
16 class BundleWriteException(Exception):
16 class BundleWriteException(Exception):
17 pass
17 pass
18
18
19 class BundleReadException(Exception):
19 class BundleReadException(Exception):
20 pass
20 pass
21
21
22 class abstractbundlestore(object):
22 class abstractbundlestore(object):
23 """Defines the interface for bundle stores.
23 """Defines the interface for bundle stores.
24
24
25 A bundle store is an entity that stores raw bundle data. It is a simple
25 A bundle store is an entity that stores raw bundle data. It is a simple
26 key-value store. However, the keys are chosen by the store. The keys can
26 key-value store. However, the keys are chosen by the store. The keys can
27 be any Python object understood by the corresponding bundle index (see
27 be any Python object understood by the corresponding bundle index (see
28 ``abstractbundleindex`` below).
28 ``abstractbundleindex`` below).
29 """
29 """
30 __metaclass__ = abc.ABCMeta
30 __metaclass__ = abc.ABCMeta
31
31
32 @abc.abstractmethod
32 @abc.abstractmethod
33 def write(self, data):
33 def write(self, data):
34 """Write bundle data to the store.
34 """Write bundle data to the store.
35
35
36 This function receives the raw data to be written as a str.
36 This function receives the raw data to be written as a str.
37 Throws BundleWriteException
37 Throws BundleWriteException
38 The key of the written data MUST be returned.
38 The key of the written data MUST be returned.
39 """
39 """
40
40
41 @abc.abstractmethod
41 @abc.abstractmethod
42 def read(self, key):
42 def read(self, key):
43 """Obtain bundle data for a key.
43 """Obtain bundle data for a key.
44
44
45 Returns None if the bundle isn't known.
45 Returns None if the bundle isn't known.
46 Throws BundleReadException
46 Throws BundleReadException
47 The returned object should be a file object supporting read()
47 The returned object should be a file object supporting read()
48 and close().
48 and close().
49 """
49 """
50
50
51 class filebundlestore(object):
51 class filebundlestore(object):
52 """bundle store in filesystem
52 """bundle store in filesystem
53
53
54 meant for storing bundles somewhere on disk and on network filesystems
54 meant for storing bundles somewhere on disk and on network filesystems
55 """
55 """
56 def __init__(self, ui, repo):
56 def __init__(self, ui, repo):
57 self.ui = ui
57 self.ui = ui
58 self.repo = repo
58 self.repo = repo
59 self.storepath = ui.configpath('scratchbranch', 'storepath')
59 self.storepath = ui.configpath('scratchbranch', 'storepath')
60 if not self.storepath:
60 if not self.storepath:
61 self.storepath = self.repo.vfs.join("scratchbranches",
61 self.storepath = self.repo.vfs.join("scratchbranches",
62 "filebundlestore")
62 "filebundlestore")
63 if not os.path.exists(self.storepath):
63 if not os.path.exists(self.storepath):
64 os.makedirs(self.storepath)
64 os.makedirs(self.storepath)
65
65
66 def _dirpath(self, hashvalue):
66 def _dirpath(self, hashvalue):
67 """First two bytes of the hash are the name of the upper
67 """First two bytes of the hash are the name of the upper
68 level directory, next two bytes are the name of the
68 level directory, next two bytes are the name of the
69 next level directory"""
69 next level directory"""
70 return os.path.join(self.storepath, hashvalue[0:2], hashvalue[2:4])
70 return os.path.join(self.storepath, hashvalue[0:2], hashvalue[2:4])
71
71
72 def _filepath(self, filename):
72 def _filepath(self, filename):
73 return os.path.join(self._dirpath(filename), filename)
73 return os.path.join(self._dirpath(filename), filename)
74
74
75 def write(self, data):
75 def write(self, data):
76 filename = hashlib.sha1(data).hexdigest()
76 filename = hashlib.sha1(data).hexdigest()
77 dirpath = self._dirpath(filename)
77 dirpath = self._dirpath(filename)
78
78
79 if not os.path.exists(dirpath):
79 if not os.path.exists(dirpath):
80 os.makedirs(dirpath)
80 os.makedirs(dirpath)
81
81
82 with open(self._filepath(filename), 'wb') as f:
82 with open(self._filepath(filename), 'wb') as f:
83 f.write(data)
83 f.write(data)
84
84
85 return filename
85 return filename
86
86
87 def read(self, key):
87 def read(self, key):
88 try:
88 try:
89 f = open(self._filepath(key), 'rb')
89 with open(self._filepath(key), 'rb') as f:
90 return f.read()
90 except IOError:
91 except IOError:
91 return None
92 return None
92
93
93 return f.read()
94
95 class externalbundlestore(abstractbundlestore):
94 class externalbundlestore(abstractbundlestore):
96 def __init__(self, put_binary, put_args, get_binary, get_args):
95 def __init__(self, put_binary, put_args, get_binary, get_args):
97 """
96 """
98 `put_binary` - path to binary file which uploads bundle to external
97 `put_binary` - path to binary file which uploads bundle to external
99 storage and prints key to stdout
98 storage and prints key to stdout
100 `put_args` - format string with additional args to `put_binary`
99 `put_args` - format string with additional args to `put_binary`
101 {filename} replacement field can be used.
100 {filename} replacement field can be used.
102 `get_binary` - path to binary file which accepts filename and key
101 `get_binary` - path to binary file which accepts filename and key
103 (in that order), downloads bundle from store and saves it to file
102 (in that order), downloads bundle from store and saves it to file
104 `get_args` - format string with additional args to `get_binary`.
103 `get_args` - format string with additional args to `get_binary`.
105 {filename} and {handle} replacement field can be used.
104 {filename} and {handle} replacement field can be used.
106 """
105 """
107
106
108 self.put_args = put_args
107 self.put_args = put_args
109 self.get_args = get_args
108 self.get_args = get_args
110 self.put_binary = put_binary
109 self.put_binary = put_binary
111 self.get_binary = get_binary
110 self.get_binary = get_binary
112
111
113 def _call_binary(self, args):
112 def _call_binary(self, args):
114 p = subprocess.Popen(
113 p = subprocess.Popen(
115 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
114 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
116 close_fds=True)
115 close_fds=True)
117 stdout, stderr = p.communicate()
116 stdout, stderr = p.communicate()
118 returncode = p.returncode
117 returncode = p.returncode
119 return returncode, stdout, stderr
118 return returncode, stdout, stderr
120
119
121 def write(self, data):
120 def write(self, data):
122 # Won't work on windows because you can't open file second time without
121 # Won't work on windows because you can't open file second time without
123 # closing it
122 # closing it
124 with NamedTemporaryFile() as temp:
123 with NamedTemporaryFile() as temp:
125 temp.write(data)
124 temp.write(data)
126 temp.flush()
125 temp.flush()
127 temp.seek(0)
126 temp.seek(0)
128 formatted_args = [arg.format(filename=temp.name)
127 formatted_args = [arg.format(filename=temp.name)
129 for arg in self.put_args]
128 for arg in self.put_args]
130 returncode, stdout, stderr = self._call_binary(
129 returncode, stdout, stderr = self._call_binary(
131 [self.put_binary] + formatted_args)
130 [self.put_binary] + formatted_args)
132
131
133 if returncode != 0:
132 if returncode != 0:
134 raise BundleWriteException(
133 raise BundleWriteException(
135 'Failed to upload to external store: %s' % stderr)
134 'Failed to upload to external store: %s' % stderr)
136 stdout_lines = stdout.splitlines()
135 stdout_lines = stdout.splitlines()
137 if len(stdout_lines) == 1:
136 if len(stdout_lines) == 1:
138 return stdout_lines[0]
137 return stdout_lines[0]
139 else:
138 else:
140 raise BundleWriteException(
139 raise BundleWriteException(
141 'Bad output from %s: %s' % (self.put_binary, stdout))
140 'Bad output from %s: %s' % (self.put_binary, stdout))
142
141
143 def read(self, handle):
142 def read(self, handle):
144 # Won't work on windows because you can't open file second time without
143 # Won't work on windows because you can't open file second time without
145 # closing it
144 # closing it
146 with NamedTemporaryFile() as temp:
145 with NamedTemporaryFile() as temp:
147 formatted_args = [arg.format(filename=temp.name, handle=handle)
146 formatted_args = [arg.format(filename=temp.name, handle=handle)
148 for arg in self.get_args]
147 for arg in self.get_args]
149 returncode, stdout, stderr = self._call_binary(
148 returncode, stdout, stderr = self._call_binary(
150 [self.get_binary] + formatted_args)
149 [self.get_binary] + formatted_args)
151
150
152 if returncode != 0:
151 if returncode != 0:
153 raise BundleReadException(
152 raise BundleReadException(
154 'Failed to download from external store: %s' % stderr)
153 'Failed to download from external store: %s' % stderr)
155 return temp.read()
154 return temp.read()
General Comments 0
You need to be logged in to leave comments. Login now