Show More
@@ -1,472 +1,471 b'' | |||||
1 | # This library is free software; you can redistribute it and/or |
|
1 | # This library is free software; you can redistribute it and/or | |
2 | # modify it under the terms of the GNU Lesser General Public |
|
2 | # modify it under the terms of the GNU Lesser General Public | |
3 | # License as published by the Free Software Foundation; either |
|
3 | # License as published by the Free Software Foundation; either | |
4 | # version 2.1 of the License, or (at your option) any later version. |
|
4 | # version 2.1 of the License, or (at your option) any later version. | |
5 | # |
|
5 | # | |
6 | # This library is distributed in the hope that it will be useful, |
|
6 | # This library is distributed in the hope that it will be useful, | |
7 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
7 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
8 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
|
8 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
9 | # Lesser General Public License for more details. |
|
9 | # Lesser General Public License for more details. | |
10 | # |
|
10 | # | |
11 | # You should have received a copy of the GNU Lesser General Public |
|
11 | # You should have received a copy of the GNU Lesser General Public | |
12 | # License along with this library; if not, see |
|
12 | # License along with this library; if not, see | |
13 | # <http://www.gnu.org/licenses/>. |
|
13 | # <http://www.gnu.org/licenses/>. | |
14 |
|
14 | |||
15 | # This file is part of urlgrabber, a high-level cross-protocol url-grabber |
|
15 | # This file is part of urlgrabber, a high-level cross-protocol url-grabber | |
16 | # Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko |
|
16 | # Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko | |
17 |
|
17 | |||
18 | # $Id: byterange.py,v 1.9 2005/02/14 21:55:07 mstenner Exp $ |
|
18 | # $Id: byterange.py,v 1.9 2005/02/14 21:55:07 mstenner Exp $ | |
19 |
|
19 | |||
20 | from __future__ import absolute_import |
|
20 | from __future__ import absolute_import | |
21 |
|
21 | |||
22 | import email |
|
22 | import email | |
23 | import ftplib |
|
23 | import ftplib | |
24 | import mimetypes |
|
24 | import mimetypes | |
25 | import os |
|
25 | import os | |
26 | import re |
|
26 | import re | |
27 | import socket |
|
27 | import socket | |
28 | import stat |
|
28 | import stat | |
29 |
|
29 | |||
30 | from . import ( |
|
30 | from . import ( | |
31 | util, |
|
31 | util, | |
32 | ) |
|
32 | ) | |
33 |
|
33 | |||
34 | urlerr = util.urlerr |
|
34 | urlerr = util.urlerr | |
35 | urlreq = util.urlreq |
|
35 | urlreq = util.urlreq | |
36 |
|
36 | |||
37 | addclosehook = urlreq.addclosehook |
|
37 | addclosehook = urlreq.addclosehook | |
38 | addinfourl = urlreq.addinfourl |
|
38 | addinfourl = urlreq.addinfourl | |
39 | splitattr = urlreq.splitattr |
|
39 | splitattr = urlreq.splitattr | |
40 | splitpasswd = urlreq.splitpasswd |
|
40 | splitpasswd = urlreq.splitpasswd | |
41 | splitport = urlreq.splitport |
|
41 | splitport = urlreq.splitport | |
42 | splituser = urlreq.splituser |
|
42 | splituser = urlreq.splituser | |
43 | unquote = urlreq.unquote |
|
43 | unquote = urlreq.unquote | |
44 |
|
44 | |||
45 | class RangeError(IOError): |
|
45 | class RangeError(IOError): | |
46 | """Error raised when an unsatisfiable range is requested.""" |
|
46 | """Error raised when an unsatisfiable range is requested.""" | |
47 | pass |
|
|||
48 |
|
47 | |||
49 | class HTTPRangeHandler(urlreq.basehandler): |
|
48 | class HTTPRangeHandler(urlreq.basehandler): | |
50 | """Handler that enables HTTP Range headers. |
|
49 | """Handler that enables HTTP Range headers. | |
51 |
|
50 | |||
52 | This was extremely simple. The Range header is a HTTP feature to |
|
51 | This was extremely simple. The Range header is a HTTP feature to | |
53 | begin with so all this class does is tell urllib2 that the |
|
52 | begin with so all this class does is tell urllib2 that the | |
54 | "206 Partial Content" response from the HTTP server is what we |
|
53 | "206 Partial Content" response from the HTTP server is what we | |
55 | expected. |
|
54 | expected. | |
56 |
|
55 | |||
57 | Example: |
|
56 | Example: | |
58 | import urllib2 |
|
57 | import urllib2 | |
59 | import byterange |
|
58 | import byterange | |
60 |
|
59 | |||
61 | range_handler = range.HTTPRangeHandler() |
|
60 | range_handler = range.HTTPRangeHandler() | |
62 | opener = urlreq.buildopener(range_handler) |
|
61 | opener = urlreq.buildopener(range_handler) | |
63 |
|
62 | |||
64 | # install it |
|
63 | # install it | |
65 | urlreq.installopener(opener) |
|
64 | urlreq.installopener(opener) | |
66 |
|
65 | |||
67 | # create Request and set Range header |
|
66 | # create Request and set Range header | |
68 | req = urlreq.request('http://www.python.org/') |
|
67 | req = urlreq.request('http://www.python.org/') | |
69 | req.header['Range'] = 'bytes=30-50' |
|
68 | req.header['Range'] = 'bytes=30-50' | |
70 | f = urlreq.urlopen(req) |
|
69 | f = urlreq.urlopen(req) | |
71 | """ |
|
70 | """ | |
72 |
|
71 | |||
73 | def http_error_206(self, req, fp, code, msg, hdrs): |
|
72 | def http_error_206(self, req, fp, code, msg, hdrs): | |
74 | # 206 Partial Content Response |
|
73 | # 206 Partial Content Response | |
75 | r = urlreq.addinfourl(fp, hdrs, req.get_full_url()) |
|
74 | r = urlreq.addinfourl(fp, hdrs, req.get_full_url()) | |
76 | r.code = code |
|
75 | r.code = code | |
77 | r.msg = msg |
|
76 | r.msg = msg | |
78 | return r |
|
77 | return r | |
79 |
|
78 | |||
80 | def http_error_416(self, req, fp, code, msg, hdrs): |
|
79 | def http_error_416(self, req, fp, code, msg, hdrs): | |
81 | # HTTP's Range Not Satisfiable error |
|
80 | # HTTP's Range Not Satisfiable error | |
82 | raise RangeError('Requested Range Not Satisfiable') |
|
81 | raise RangeError('Requested Range Not Satisfiable') | |
83 |
|
82 | |||
84 | class RangeableFileObject(object): |
|
83 | class RangeableFileObject(object): | |
85 | """File object wrapper to enable raw range handling. |
|
84 | """File object wrapper to enable raw range handling. | |
86 | This was implemented primarily for handling range |
|
85 | This was implemented primarily for handling range | |
87 | specifications for file:// urls. This object effectively makes |
|
86 | specifications for file:// urls. This object effectively makes | |
88 | a file object look like it consists only of a range of bytes in |
|
87 | a file object look like it consists only of a range of bytes in | |
89 | the stream. |
|
88 | the stream. | |
90 |
|
89 | |||
91 | Examples: |
|
90 | Examples: | |
92 | # expose 10 bytes, starting at byte position 20, from |
|
91 | # expose 10 bytes, starting at byte position 20, from | |
93 | # /etc/aliases. |
|
92 | # /etc/aliases. | |
94 | >>> fo = RangeableFileObject(file(b'/etc/passwd', b'r'), (20,30)) |
|
93 | >>> fo = RangeableFileObject(file(b'/etc/passwd', b'r'), (20,30)) | |
95 | # seek seeks within the range (to position 23 in this case) |
|
94 | # seek seeks within the range (to position 23 in this case) | |
96 | >>> fo.seek(3) |
|
95 | >>> fo.seek(3) | |
97 | # tell tells where your at _within the range_ (position 3 in |
|
96 | # tell tells where your at _within the range_ (position 3 in | |
98 | # this case) |
|
97 | # this case) | |
99 | >>> fo.tell() |
|
98 | >>> fo.tell() | |
100 | # read EOFs if an attempt is made to read past the last |
|
99 | # read EOFs if an attempt is made to read past the last | |
101 | # byte in the range. the following will return only 7 bytes. |
|
100 | # byte in the range. the following will return only 7 bytes. | |
102 | >>> fo.read(30) |
|
101 | >>> fo.read(30) | |
103 | """ |
|
102 | """ | |
104 |
|
103 | |||
105 | def __init__(self, fo, rangetup): |
|
104 | def __init__(self, fo, rangetup): | |
106 | """Create a RangeableFileObject. |
|
105 | """Create a RangeableFileObject. | |
107 | fo -- a file like object. only the read() method need be |
|
106 | fo -- a file like object. only the read() method need be | |
108 | supported but supporting an optimized seek() is |
|
107 | supported but supporting an optimized seek() is | |
109 | preferable. |
|
108 | preferable. | |
110 | rangetup -- a (firstbyte,lastbyte) tuple specifying the range |
|
109 | rangetup -- a (firstbyte,lastbyte) tuple specifying the range | |
111 | to work over. |
|
110 | to work over. | |
112 | The file object provided is assumed to be at byte offset 0. |
|
111 | The file object provided is assumed to be at byte offset 0. | |
113 | """ |
|
112 | """ | |
114 | self.fo = fo |
|
113 | self.fo = fo | |
115 | (self.firstbyte, self.lastbyte) = range_tuple_normalize(rangetup) |
|
114 | (self.firstbyte, self.lastbyte) = range_tuple_normalize(rangetup) | |
116 | self.realpos = 0 |
|
115 | self.realpos = 0 | |
117 | self._do_seek(self.firstbyte) |
|
116 | self._do_seek(self.firstbyte) | |
118 |
|
117 | |||
119 | def __getattr__(self, name): |
|
118 | def __getattr__(self, name): | |
120 | """This effectively allows us to wrap at the instance level. |
|
119 | """This effectively allows us to wrap at the instance level. | |
121 | Any attribute not found in _this_ object will be searched for |
|
120 | Any attribute not found in _this_ object will be searched for | |
122 | in self.fo. This includes methods.""" |
|
121 | in self.fo. This includes methods.""" | |
123 | return getattr(self.fo, name) |
|
122 | return getattr(self.fo, name) | |
124 |
|
123 | |||
125 | def tell(self): |
|
124 | def tell(self): | |
126 | """Return the position within the range. |
|
125 | """Return the position within the range. | |
127 | This is different from fo.seek in that position 0 is the |
|
126 | This is different from fo.seek in that position 0 is the | |
128 | first byte position of the range tuple. For example, if |
|
127 | first byte position of the range tuple. For example, if | |
129 | this object was created with a range tuple of (500,899), |
|
128 | this object was created with a range tuple of (500,899), | |
130 | tell() will return 0 when at byte position 500 of the file. |
|
129 | tell() will return 0 when at byte position 500 of the file. | |
131 | """ |
|
130 | """ | |
132 | return (self.realpos - self.firstbyte) |
|
131 | return (self.realpos - self.firstbyte) | |
133 |
|
132 | |||
134 | def seek(self, offset, whence=0): |
|
133 | def seek(self, offset, whence=0): | |
135 | """Seek within the byte range. |
|
134 | """Seek within the byte range. | |
136 | Positioning is identical to that described under tell(). |
|
135 | Positioning is identical to that described under tell(). | |
137 | """ |
|
136 | """ | |
138 | assert whence in (0, 1, 2) |
|
137 | assert whence in (0, 1, 2) | |
139 | if whence == 0: # absolute seek |
|
138 | if whence == 0: # absolute seek | |
140 | realoffset = self.firstbyte + offset |
|
139 | realoffset = self.firstbyte + offset | |
141 | elif whence == 1: # relative seek |
|
140 | elif whence == 1: # relative seek | |
142 | realoffset = self.realpos + offset |
|
141 | realoffset = self.realpos + offset | |
143 | elif whence == 2: # absolute from end of file |
|
142 | elif whence == 2: # absolute from end of file | |
144 | # XXX: are we raising the right Error here? |
|
143 | # XXX: are we raising the right Error here? | |
145 | raise IOError('seek from end of file not supported.') |
|
144 | raise IOError('seek from end of file not supported.') | |
146 |
|
145 | |||
147 | # do not allow seek past lastbyte in range |
|
146 | # do not allow seek past lastbyte in range | |
148 | if self.lastbyte and (realoffset >= self.lastbyte): |
|
147 | if self.lastbyte and (realoffset >= self.lastbyte): | |
149 | realoffset = self.lastbyte |
|
148 | realoffset = self.lastbyte | |
150 |
|
149 | |||
151 | self._do_seek(realoffset - self.realpos) |
|
150 | self._do_seek(realoffset - self.realpos) | |
152 |
|
151 | |||
153 | def read(self, size=-1): |
|
152 | def read(self, size=-1): | |
154 | """Read within the range. |
|
153 | """Read within the range. | |
155 | This method will limit the size read based on the range. |
|
154 | This method will limit the size read based on the range. | |
156 | """ |
|
155 | """ | |
157 | size = self._calc_read_size(size) |
|
156 | size = self._calc_read_size(size) | |
158 | rslt = self.fo.read(size) |
|
157 | rslt = self.fo.read(size) | |
159 | self.realpos += len(rslt) |
|
158 | self.realpos += len(rslt) | |
160 | return rslt |
|
159 | return rslt | |
161 |
|
160 | |||
162 | def readline(self, size=-1): |
|
161 | def readline(self, size=-1): | |
163 | """Read lines within the range. |
|
162 | """Read lines within the range. | |
164 | This method will limit the size read based on the range. |
|
163 | This method will limit the size read based on the range. | |
165 | """ |
|
164 | """ | |
166 | size = self._calc_read_size(size) |
|
165 | size = self._calc_read_size(size) | |
167 | rslt = self.fo.readline(size) |
|
166 | rslt = self.fo.readline(size) | |
168 | self.realpos += len(rslt) |
|
167 | self.realpos += len(rslt) | |
169 | return rslt |
|
168 | return rslt | |
170 |
|
169 | |||
171 | def _calc_read_size(self, size): |
|
170 | def _calc_read_size(self, size): | |
172 | """Handles calculating the amount of data to read based on |
|
171 | """Handles calculating the amount of data to read based on | |
173 | the range. |
|
172 | the range. | |
174 | """ |
|
173 | """ | |
175 | if self.lastbyte: |
|
174 | if self.lastbyte: | |
176 | if size > -1: |
|
175 | if size > -1: | |
177 | if ((self.realpos + size) >= self.lastbyte): |
|
176 | if ((self.realpos + size) >= self.lastbyte): | |
178 | size = (self.lastbyte - self.realpos) |
|
177 | size = (self.lastbyte - self.realpos) | |
179 | else: |
|
178 | else: | |
180 | size = (self.lastbyte - self.realpos) |
|
179 | size = (self.lastbyte - self.realpos) | |
181 | return size |
|
180 | return size | |
182 |
|
181 | |||
183 | def _do_seek(self, offset): |
|
182 | def _do_seek(self, offset): | |
184 | """Seek based on whether wrapped object supports seek(). |
|
183 | """Seek based on whether wrapped object supports seek(). | |
185 | offset is relative to the current position (self.realpos). |
|
184 | offset is relative to the current position (self.realpos). | |
186 | """ |
|
185 | """ | |
187 | assert offset >= 0 |
|
186 | assert offset >= 0 | |
188 | seek = getattr(self.fo, 'seek', self._poor_mans_seek) |
|
187 | seek = getattr(self.fo, 'seek', self._poor_mans_seek) | |
189 | seek(self.realpos + offset) |
|
188 | seek(self.realpos + offset) | |
190 | self.realpos += offset |
|
189 | self.realpos += offset | |
191 |
|
190 | |||
192 | def _poor_mans_seek(self, offset): |
|
191 | def _poor_mans_seek(self, offset): | |
193 | """Seek by calling the wrapped file objects read() method. |
|
192 | """Seek by calling the wrapped file objects read() method. | |
194 | This is used for file like objects that do not have native |
|
193 | This is used for file like objects that do not have native | |
195 | seek support. The wrapped objects read() method is called |
|
194 | seek support. The wrapped objects read() method is called | |
196 | to manually seek to the desired position. |
|
195 | to manually seek to the desired position. | |
197 | offset -- read this number of bytes from the wrapped |
|
196 | offset -- read this number of bytes from the wrapped | |
198 | file object. |
|
197 | file object. | |
199 | raise RangeError if we encounter EOF before reaching the |
|
198 | raise RangeError if we encounter EOF before reaching the | |
200 | specified offset. |
|
199 | specified offset. | |
201 | """ |
|
200 | """ | |
202 | pos = 0 |
|
201 | pos = 0 | |
203 | bufsize = 1024 |
|
202 | bufsize = 1024 | |
204 | while pos < offset: |
|
203 | while pos < offset: | |
205 | if (pos + bufsize) > offset: |
|
204 | if (pos + bufsize) > offset: | |
206 | bufsize = offset - pos |
|
205 | bufsize = offset - pos | |
207 | buf = self.fo.read(bufsize) |
|
206 | buf = self.fo.read(bufsize) | |
208 | if len(buf) != bufsize: |
|
207 | if len(buf) != bufsize: | |
209 | raise RangeError('Requested Range Not Satisfiable') |
|
208 | raise RangeError('Requested Range Not Satisfiable') | |
210 | pos += bufsize |
|
209 | pos += bufsize | |
211 |
|
210 | |||
212 | class FileRangeHandler(urlreq.filehandler): |
|
211 | class FileRangeHandler(urlreq.filehandler): | |
213 | """FileHandler subclass that adds Range support. |
|
212 | """FileHandler subclass that adds Range support. | |
214 | This class handles Range headers exactly like an HTTP |
|
213 | This class handles Range headers exactly like an HTTP | |
215 | server would. |
|
214 | server would. | |
216 | """ |
|
215 | """ | |
217 | def open_local_file(self, req): |
|
216 | def open_local_file(self, req): | |
218 | host = req.get_host() |
|
217 | host = req.get_host() | |
219 | file = req.get_selector() |
|
218 | file = req.get_selector() | |
220 | localfile = urlreq.url2pathname(file) |
|
219 | localfile = urlreq.url2pathname(file) | |
221 | stats = os.stat(localfile) |
|
220 | stats = os.stat(localfile) | |
222 | size = stats[stat.ST_SIZE] |
|
221 | size = stats[stat.ST_SIZE] | |
223 | modified = email.Utils.formatdate(stats[stat.ST_MTIME]) |
|
222 | modified = email.Utils.formatdate(stats[stat.ST_MTIME]) | |
224 | mtype = mimetypes.guess_type(file)[0] |
|
223 | mtype = mimetypes.guess_type(file)[0] | |
225 | if host: |
|
224 | if host: | |
226 | host, port = urlreq.splitport(host) |
|
225 | host, port = urlreq.splitport(host) | |
227 | if port or socket.gethostbyname(host) not in self.get_names(): |
|
226 | if port or socket.gethostbyname(host) not in self.get_names(): | |
228 | raise urlerr.urlerror('file not on local host') |
|
227 | raise urlerr.urlerror('file not on local host') | |
229 | fo = open(localfile,'rb') |
|
228 | fo = open(localfile,'rb') | |
230 | brange = req.headers.get('Range', None) |
|
229 | brange = req.headers.get('Range', None) | |
231 | brange = range_header_to_tuple(brange) |
|
230 | brange = range_header_to_tuple(brange) | |
232 | assert brange != () |
|
231 | assert brange != () | |
233 | if brange: |
|
232 | if brange: | |
234 | (fb, lb) = brange |
|
233 | (fb, lb) = brange | |
235 | if lb == '': |
|
234 | if lb == '': | |
236 | lb = size |
|
235 | lb = size | |
237 | if fb < 0 or fb > size or lb > size: |
|
236 | if fb < 0 or fb > size or lb > size: | |
238 | raise RangeError('Requested Range Not Satisfiable') |
|
237 | raise RangeError('Requested Range Not Satisfiable') | |
239 | size = (lb - fb) |
|
238 | size = (lb - fb) | |
240 | fo = RangeableFileObject(fo, (fb, lb)) |
|
239 | fo = RangeableFileObject(fo, (fb, lb)) | |
241 | headers = email.message_from_string( |
|
240 | headers = email.message_from_string( | |
242 | 'Content-Type: %s\nContent-Length: %d\nLast-Modified: %s\n' % |
|
241 | 'Content-Type: %s\nContent-Length: %d\nLast-Modified: %s\n' % | |
243 | (mtype or 'text/plain', size, modified)) |
|
242 | (mtype or 'text/plain', size, modified)) | |
244 | return urlreq.addinfourl(fo, headers, 'file:'+file) |
|
243 | return urlreq.addinfourl(fo, headers, 'file:'+file) | |
245 |
|
244 | |||
246 |
|
245 | |||
247 | # FTP Range Support |
|
246 | # FTP Range Support | |
248 | # Unfortunately, a large amount of base FTP code had to be copied |
|
247 | # Unfortunately, a large amount of base FTP code had to be copied | |
249 | # from urllib and urllib2 in order to insert the FTP REST command. |
|
248 | # from urllib and urllib2 in order to insert the FTP REST command. | |
250 | # Code modifications for range support have been commented as |
|
249 | # Code modifications for range support have been commented as | |
251 | # follows: |
|
250 | # follows: | |
252 | # -- range support modifications start/end here |
|
251 | # -- range support modifications start/end here | |
253 |
|
252 | |||
254 | class FTPRangeHandler(urlreq.ftphandler): |
|
253 | class FTPRangeHandler(urlreq.ftphandler): | |
255 | def ftp_open(self, req): |
|
254 | def ftp_open(self, req): | |
256 | host = req.get_host() |
|
255 | host = req.get_host() | |
257 | if not host: |
|
256 | if not host: | |
258 | raise IOError('ftp error', 'no host given') |
|
257 | raise IOError('ftp error', 'no host given') | |
259 | host, port = splitport(host) |
|
258 | host, port = splitport(host) | |
260 | if port is None: |
|
259 | if port is None: | |
261 | port = ftplib.FTP_PORT |
|
260 | port = ftplib.FTP_PORT | |
262 | else: |
|
261 | else: | |
263 | port = int(port) |
|
262 | port = int(port) | |
264 |
|
263 | |||
265 | # username/password handling |
|
264 | # username/password handling | |
266 | user, host = splituser(host) |
|
265 | user, host = splituser(host) | |
267 | if user: |
|
266 | if user: | |
268 | user, passwd = splitpasswd(user) |
|
267 | user, passwd = splitpasswd(user) | |
269 | else: |
|
268 | else: | |
270 | passwd = None |
|
269 | passwd = None | |
271 | host = unquote(host) |
|
270 | host = unquote(host) | |
272 | user = unquote(user or '') |
|
271 | user = unquote(user or '') | |
273 | passwd = unquote(passwd or '') |
|
272 | passwd = unquote(passwd or '') | |
274 |
|
273 | |||
275 | try: |
|
274 | try: | |
276 | host = socket.gethostbyname(host) |
|
275 | host = socket.gethostbyname(host) | |
277 | except socket.error as msg: |
|
276 | except socket.error as msg: | |
278 | raise urlerr.urlerror(msg) |
|
277 | raise urlerr.urlerror(msg) | |
279 | path, attrs = splitattr(req.get_selector()) |
|
278 | path, attrs = splitattr(req.get_selector()) | |
280 | dirs = path.split('/') |
|
279 | dirs = path.split('/') | |
281 | dirs = map(unquote, dirs) |
|
280 | dirs = map(unquote, dirs) | |
282 | dirs, file = dirs[:-1], dirs[-1] |
|
281 | dirs, file = dirs[:-1], dirs[-1] | |
283 | if dirs and not dirs[0]: |
|
282 | if dirs and not dirs[0]: | |
284 | dirs = dirs[1:] |
|
283 | dirs = dirs[1:] | |
285 | try: |
|
284 | try: | |
286 | fw = self.connect_ftp(user, passwd, host, port, dirs) |
|
285 | fw = self.connect_ftp(user, passwd, host, port, dirs) | |
287 | if file: |
|
286 | if file: | |
288 | type = 'I' |
|
287 | type = 'I' | |
289 | else: |
|
288 | else: | |
290 | type = 'D' |
|
289 | type = 'D' | |
291 |
|
290 | |||
292 | for attr in attrs: |
|
291 | for attr in attrs: | |
293 | attr, value = splitattr(attr) |
|
292 | attr, value = splitattr(attr) | |
294 | if attr.lower() == 'type' and \ |
|
293 | if attr.lower() == 'type' and \ | |
295 | value in ('a', 'A', 'i', 'I', 'd', 'D'): |
|
294 | value in ('a', 'A', 'i', 'I', 'd', 'D'): | |
296 | type = value.upper() |
|
295 | type = value.upper() | |
297 |
|
296 | |||
298 | # -- range support modifications start here |
|
297 | # -- range support modifications start here | |
299 | rest = None |
|
298 | rest = None | |
300 | range_tup = range_header_to_tuple(req.headers.get('Range', None)) |
|
299 | range_tup = range_header_to_tuple(req.headers.get('Range', None)) | |
301 | assert range_tup != () |
|
300 | assert range_tup != () | |
302 | if range_tup: |
|
301 | if range_tup: | |
303 | (fb, lb) = range_tup |
|
302 | (fb, lb) = range_tup | |
304 | if fb > 0: |
|
303 | if fb > 0: | |
305 | rest = fb |
|
304 | rest = fb | |
306 | # -- range support modifications end here |
|
305 | # -- range support modifications end here | |
307 |
|
306 | |||
308 | fp, retrlen = fw.retrfile(file, type, rest) |
|
307 | fp, retrlen = fw.retrfile(file, type, rest) | |
309 |
|
308 | |||
310 | # -- range support modifications start here |
|
309 | # -- range support modifications start here | |
311 | if range_tup: |
|
310 | if range_tup: | |
312 | (fb, lb) = range_tup |
|
311 | (fb, lb) = range_tup | |
313 | if lb == '': |
|
312 | if lb == '': | |
314 | if retrlen is None or retrlen == 0: |
|
313 | if retrlen is None or retrlen == 0: | |
315 | raise RangeError('Requested Range Not Satisfiable due' |
|
314 | raise RangeError('Requested Range Not Satisfiable due' | |
316 | ' to unobtainable file length.') |
|
315 | ' to unobtainable file length.') | |
317 | lb = retrlen |
|
316 | lb = retrlen | |
318 | retrlen = lb - fb |
|
317 | retrlen = lb - fb | |
319 | if retrlen < 0: |
|
318 | if retrlen < 0: | |
320 | # beginning of range is larger than file |
|
319 | # beginning of range is larger than file | |
321 | raise RangeError('Requested Range Not Satisfiable') |
|
320 | raise RangeError('Requested Range Not Satisfiable') | |
322 | else: |
|
321 | else: | |
323 | retrlen = lb - fb |
|
322 | retrlen = lb - fb | |
324 | fp = RangeableFileObject(fp, (0, retrlen)) |
|
323 | fp = RangeableFileObject(fp, (0, retrlen)) | |
325 | # -- range support modifications end here |
|
324 | # -- range support modifications end here | |
326 |
|
325 | |||
327 | headers = "" |
|
326 | headers = "" | |
328 | mtype = mimetypes.guess_type(req.get_full_url())[0] |
|
327 | mtype = mimetypes.guess_type(req.get_full_url())[0] | |
329 | if mtype: |
|
328 | if mtype: | |
330 | headers += "Content-Type: %s\n" % mtype |
|
329 | headers += "Content-Type: %s\n" % mtype | |
331 | if retrlen is not None and retrlen >= 0: |
|
330 | if retrlen is not None and retrlen >= 0: | |
332 | headers += "Content-Length: %d\n" % retrlen |
|
331 | headers += "Content-Length: %d\n" % retrlen | |
333 | headers = email.message_from_string(headers) |
|
332 | headers = email.message_from_string(headers) | |
334 | return addinfourl(fp, headers, req.get_full_url()) |
|
333 | return addinfourl(fp, headers, req.get_full_url()) | |
335 | except ftplib.all_errors as msg: |
|
334 | except ftplib.all_errors as msg: | |
336 | raise IOError('ftp error', msg) |
|
335 | raise IOError('ftp error', msg) | |
337 |
|
336 | |||
338 | def connect_ftp(self, user, passwd, host, port, dirs): |
|
337 | def connect_ftp(self, user, passwd, host, port, dirs): | |
339 | fw = ftpwrapper(user, passwd, host, port, dirs) |
|
338 | fw = ftpwrapper(user, passwd, host, port, dirs) | |
340 | return fw |
|
339 | return fw | |
341 |
|
340 | |||
342 | class ftpwrapper(urlreq.ftpwrapper): |
|
341 | class ftpwrapper(urlreq.ftpwrapper): | |
343 | # range support note: |
|
342 | # range support note: | |
344 | # this ftpwrapper code is copied directly from |
|
343 | # this ftpwrapper code is copied directly from | |
345 | # urllib. The only enhancement is to add the rest |
|
344 | # urllib. The only enhancement is to add the rest | |
346 | # argument and pass it on to ftp.ntransfercmd |
|
345 | # argument and pass it on to ftp.ntransfercmd | |
347 | def retrfile(self, file, type, rest=None): |
|
346 | def retrfile(self, file, type, rest=None): | |
348 | self.endtransfer() |
|
347 | self.endtransfer() | |
349 | if type in ('d', 'D'): |
|
348 | if type in ('d', 'D'): | |
350 | cmd = 'TYPE A' |
|
349 | cmd = 'TYPE A' | |
351 | isdir = 1 |
|
350 | isdir = 1 | |
352 | else: |
|
351 | else: | |
353 | cmd = 'TYPE ' + type |
|
352 | cmd = 'TYPE ' + type | |
354 | isdir = 0 |
|
353 | isdir = 0 | |
355 | try: |
|
354 | try: | |
356 | self.ftp.voidcmd(cmd) |
|
355 | self.ftp.voidcmd(cmd) | |
357 | except ftplib.all_errors: |
|
356 | except ftplib.all_errors: | |
358 | self.init() |
|
357 | self.init() | |
359 | self.ftp.voidcmd(cmd) |
|
358 | self.ftp.voidcmd(cmd) | |
360 | conn = None |
|
359 | conn = None | |
361 | if file and not isdir: |
|
360 | if file and not isdir: | |
362 | # Use nlst to see if the file exists at all |
|
361 | # Use nlst to see if the file exists at all | |
363 | try: |
|
362 | try: | |
364 | self.ftp.nlst(file) |
|
363 | self.ftp.nlst(file) | |
365 | except ftplib.error_perm as reason: |
|
364 | except ftplib.error_perm as reason: | |
366 | raise IOError('ftp error', reason) |
|
365 | raise IOError('ftp error', reason) | |
367 | # Restore the transfer mode! |
|
366 | # Restore the transfer mode! | |
368 | self.ftp.voidcmd(cmd) |
|
367 | self.ftp.voidcmd(cmd) | |
369 | # Try to retrieve as a file |
|
368 | # Try to retrieve as a file | |
370 | try: |
|
369 | try: | |
371 | cmd = 'RETR ' + file |
|
370 | cmd = 'RETR ' + file | |
372 | conn = self.ftp.ntransfercmd(cmd, rest) |
|
371 | conn = self.ftp.ntransfercmd(cmd, rest) | |
373 | except ftplib.error_perm as reason: |
|
372 | except ftplib.error_perm as reason: | |
374 | if str(reason).startswith('501'): |
|
373 | if str(reason).startswith('501'): | |
375 | # workaround for REST not supported error |
|
374 | # workaround for REST not supported error | |
376 | fp, retrlen = self.retrfile(file, type) |
|
375 | fp, retrlen = self.retrfile(file, type) | |
377 | fp = RangeableFileObject(fp, (rest,'')) |
|
376 | fp = RangeableFileObject(fp, (rest,'')) | |
378 | return (fp, retrlen) |
|
377 | return (fp, retrlen) | |
379 | elif not str(reason).startswith('550'): |
|
378 | elif not str(reason).startswith('550'): | |
380 | raise IOError('ftp error', reason) |
|
379 | raise IOError('ftp error', reason) | |
381 | if not conn: |
|
380 | if not conn: | |
382 | # Set transfer mode to ASCII! |
|
381 | # Set transfer mode to ASCII! | |
383 | self.ftp.voidcmd('TYPE A') |
|
382 | self.ftp.voidcmd('TYPE A') | |
384 | # Try a directory listing |
|
383 | # Try a directory listing | |
385 | if file: |
|
384 | if file: | |
386 | cmd = 'LIST ' + file |
|
385 | cmd = 'LIST ' + file | |
387 | else: |
|
386 | else: | |
388 | cmd = 'LIST' |
|
387 | cmd = 'LIST' | |
389 | conn = self.ftp.ntransfercmd(cmd) |
|
388 | conn = self.ftp.ntransfercmd(cmd) | |
390 | self.busy = 1 |
|
389 | self.busy = 1 | |
391 | # Pass back both a suitably decorated object and a retrieval length |
|
390 | # Pass back both a suitably decorated object and a retrieval length | |
392 | return (addclosehook(conn[0].makefile('rb'), |
|
391 | return (addclosehook(conn[0].makefile('rb'), | |
393 | self.endtransfer), conn[1]) |
|
392 | self.endtransfer), conn[1]) | |
394 |
|
393 | |||
395 |
|
394 | |||
396 | #################################################################### |
|
395 | #################################################################### | |
397 | # Range Tuple Functions |
|
396 | # Range Tuple Functions | |
398 | # XXX: These range tuple functions might go better in a class. |
|
397 | # XXX: These range tuple functions might go better in a class. | |
399 |
|
398 | |||
400 | _rangere = None |
|
399 | _rangere = None | |
401 | def range_header_to_tuple(range_header): |
|
400 | def range_header_to_tuple(range_header): | |
402 | """Get a (firstbyte,lastbyte) tuple from a Range header value. |
|
401 | """Get a (firstbyte,lastbyte) tuple from a Range header value. | |
403 |
|
402 | |||
404 | Range headers have the form "bytes=<firstbyte>-<lastbyte>". This |
|
403 | Range headers have the form "bytes=<firstbyte>-<lastbyte>". This | |
405 | function pulls the firstbyte and lastbyte values and returns |
|
404 | function pulls the firstbyte and lastbyte values and returns | |
406 | a (firstbyte,lastbyte) tuple. If lastbyte is not specified in |
|
405 | a (firstbyte,lastbyte) tuple. If lastbyte is not specified in | |
407 | the header value, it is returned as an empty string in the |
|
406 | the header value, it is returned as an empty string in the | |
408 | tuple. |
|
407 | tuple. | |
409 |
|
408 | |||
410 | Return None if range_header is None |
|
409 | Return None if range_header is None | |
411 | Return () if range_header does not conform to the range spec |
|
410 | Return () if range_header does not conform to the range spec | |
412 | pattern. |
|
411 | pattern. | |
413 |
|
412 | |||
414 | """ |
|
413 | """ | |
415 | global _rangere |
|
414 | global _rangere | |
416 | if range_header is None: |
|
415 | if range_header is None: | |
417 | return None |
|
416 | return None | |
418 | if _rangere is None: |
|
417 | if _rangere is None: | |
419 | _rangere = re.compile(r'^bytes=(\d{1,})-(\d*)') |
|
418 | _rangere = re.compile(r'^bytes=(\d{1,})-(\d*)') | |
420 | match = _rangere.match(range_header) |
|
419 | match = _rangere.match(range_header) | |
421 | if match: |
|
420 | if match: | |
422 | tup = range_tuple_normalize(match.group(1, 2)) |
|
421 | tup = range_tuple_normalize(match.group(1, 2)) | |
423 | if tup and tup[1]: |
|
422 | if tup and tup[1]: | |
424 | tup = (tup[0], tup[1]+1) |
|
423 | tup = (tup[0], tup[1]+1) | |
425 | return tup |
|
424 | return tup | |
426 | return () |
|
425 | return () | |
427 |
|
426 | |||
428 | def range_tuple_to_header(range_tup): |
|
427 | def range_tuple_to_header(range_tup): | |
429 | """Convert a range tuple to a Range header value. |
|
428 | """Convert a range tuple to a Range header value. | |
430 | Return a string of the form "bytes=<firstbyte>-<lastbyte>" or None |
|
429 | Return a string of the form "bytes=<firstbyte>-<lastbyte>" or None | |
431 | if no range is needed. |
|
430 | if no range is needed. | |
432 | """ |
|
431 | """ | |
433 | if range_tup is None: |
|
432 | if range_tup is None: | |
434 | return None |
|
433 | return None | |
435 | range_tup = range_tuple_normalize(range_tup) |
|
434 | range_tup = range_tuple_normalize(range_tup) | |
436 | if range_tup: |
|
435 | if range_tup: | |
437 | if range_tup[1]: |
|
436 | if range_tup[1]: | |
438 | range_tup = (range_tup[0], range_tup[1] - 1) |
|
437 | range_tup = (range_tup[0], range_tup[1] - 1) | |
439 | return 'bytes=%s-%s' % range_tup |
|
438 | return 'bytes=%s-%s' % range_tup | |
440 |
|
439 | |||
441 | def range_tuple_normalize(range_tup): |
|
440 | def range_tuple_normalize(range_tup): | |
442 | """Normalize a (first_byte,last_byte) range tuple. |
|
441 | """Normalize a (first_byte,last_byte) range tuple. | |
443 | Return a tuple whose first element is guaranteed to be an int |
|
442 | Return a tuple whose first element is guaranteed to be an int | |
444 | and whose second element will be '' (meaning: the last byte) or |
|
443 | and whose second element will be '' (meaning: the last byte) or | |
445 | an int. Finally, return None if the normalized tuple == (0,'') |
|
444 | an int. Finally, return None if the normalized tuple == (0,'') | |
446 | as that is equivalent to retrieving the entire file. |
|
445 | as that is equivalent to retrieving the entire file. | |
447 | """ |
|
446 | """ | |
448 | if range_tup is None: |
|
447 | if range_tup is None: | |
449 | return None |
|
448 | return None | |
450 | # handle first byte |
|
449 | # handle first byte | |
451 | fb = range_tup[0] |
|
450 | fb = range_tup[0] | |
452 | if fb in (None, ''): |
|
451 | if fb in (None, ''): | |
453 | fb = 0 |
|
452 | fb = 0 | |
454 | else: |
|
453 | else: | |
455 | fb = int(fb) |
|
454 | fb = int(fb) | |
456 | # handle last byte |
|
455 | # handle last byte | |
457 | try: |
|
456 | try: | |
458 | lb = range_tup[1] |
|
457 | lb = range_tup[1] | |
459 | except IndexError: |
|
458 | except IndexError: | |
460 | lb = '' |
|
459 | lb = '' | |
461 | else: |
|
460 | else: | |
462 | if lb is None: |
|
461 | if lb is None: | |
463 | lb = '' |
|
462 | lb = '' | |
464 | elif lb != '': |
|
463 | elif lb != '': | |
465 | lb = int(lb) |
|
464 | lb = int(lb) | |
466 | # check if range is over the entire file |
|
465 | # check if range is over the entire file | |
467 | if (fb, lb) == (0, ''): |
|
466 | if (fb, lb) == (0, ''): | |
468 | return None |
|
467 | return None | |
469 | # check that the range is valid |
|
468 | # check that the range is valid | |
470 | if lb < fb: |
|
469 | if lb < fb: | |
471 | raise RangeError('Invalid byte range: %s-%s' % (fb, lb)) |
|
470 | raise RangeError('Invalid byte range: %s-%s' % (fb, lb)) | |
472 | return (fb, lb) |
|
471 | return (fb, lb) |
General Comments 0
You need to be logged in to leave comments.
Login now