Show More
@@ -15,7 +15,7 b' hide platform-specific details from the ' | |||||
15 |
|
15 | |||
16 | from i18n import _ |
|
16 | from i18n import _ | |
17 | import error, osutil, encoding |
|
17 | import error, osutil, encoding | |
18 |
import |
|
18 | import errno, re, shutil, sys, tempfile, traceback | |
19 | import os, stat, time, calendar, textwrap, unicodedata, signal |
|
19 | import os, stat, time, calendar, textwrap, unicodedata, signal | |
20 | import imp |
|
20 | import imp | |
21 |
|
21 | |||
@@ -909,31 +909,36 b' class chunkbuffer(object):' | |||||
909 | """in_iter is the iterator that's iterating over the input chunks. |
|
909 | """in_iter is the iterator that's iterating over the input chunks. | |
910 | targetsize is how big a buffer to try to maintain.""" |
|
910 | targetsize is how big a buffer to try to maintain.""" | |
911 | self.iter = iter(in_iter) |
|
911 | self.iter = iter(in_iter) | |
912 |
self. |
|
912 | self._queue = [] | |
913 | self.targetsize = 2**16 |
|
|||
914 |
|
913 | |||
915 | def read(self, l): |
|
914 | def read(self, l): | |
916 | """Read L bytes of data from the iterator of chunks of data. |
|
915 | """Read L bytes of data from the iterator of chunks of data. | |
917 | Returns less than L bytes if the iterator runs dry.""" |
|
916 | Returns less than L bytes if the iterator runs dry.""" | |
918 | if l > len(self.buf) and self.iter: |
|
917 | left = l | |
919 | # Clamp to a multiple of self.targetsize |
|
918 | buf = '' | |
920 | targetsize = max(l, self.targetsize) |
|
919 | queue = self._queue | |
921 | collector = cStringIO.StringIO() |
|
920 | while left > 0: | |
922 | collector.write(self.buf) |
|
921 | # refill the queue | |
923 | collected = len(self.buf) |
|
922 | if not queue: | |
|
923 | target = 2**18 | |||
924 | for chunk in self.iter: |
|
924 | for chunk in self.iter: | |
925 |
|
|
925 | queue.append(chunk) | |
926 |
|
|
926 | target -= len(chunk) | |
927 |
if |
|
927 | if target <= 0: | |
|
928 | break | |||
|
929 | if not queue: | |||
928 | break |
|
930 | break | |
929 | if collected < targetsize: |
|
931 | ||
930 | self.iter = False |
|
932 | chunk = queue.pop(0) | |
931 |
|
|
933 | left -= len(chunk) | |
932 |
if le |
|
934 | if left < 0: | |
933 | s, self.buf = str(self.buf), '' |
|
935 | queue.insert(0, chunk[left:]) | |
|
936 | buf += chunk[:left] | |||
934 | else: |
|
937 | else: | |
935 | s, self.buf = self.buf[:l], buffer(self.buf, l) |
|
938 | buf += chunk | |
936 | return s |
|
939 | ||
|
940 | return buf | |||
|
941 | ||||
937 |
|
942 | |||
938 | def filechunkiter(f, size=65536, limit=None): |
|
943 | def filechunkiter(f, size=65536, limit=None): | |
939 | """Create a generator that produces the data in the file size |
|
944 | """Create a generator that produces the data in the file size |
General Comments 0
You need to be logged in to leave comments.
Login now