Show More
@@ -2037,3 +2037,27 b' helps performance.' | |||
|
2037 | 2037 | Number of CPUs to use for parallel operations. A zero or |
|
2038 | 2038 | negative value is treated as ``use the default``. |
|
2039 | 2039 | (default: 4 or the number of CPUs on the system, whichever is larger) |
|
2040 | ||
|
2041 | ``backgroundclose`` | |
|
2042 | Whether to enable closing file handles on background threads during certain | |
|
2043 | operations. Some platforms aren't very efficient at closing file | |
|
2044 | handles that have been written or appened to. By performing file closing | |
|
2045 | on background threads, file write rate can increase substantially. | |
|
2046 | (default: true on Windows, false elsewhere) | |
|
2047 | ||
|
2048 | ``backgroundcloseminfilecount`` | |
|
2049 | Minimum number of files required to trigger background file closing. | |
|
2050 | Operations not writing this many files won't start background close | |
|
2051 | threads. | |
|
2052 | (default: 2048) | |
|
2053 | ||
|
2054 | ``backgroundclosemaxqueue`` | |
|
2055 | The maximum number of opened file handles waiting to be closed in the | |
|
2056 | background. This option only has an effect if ``backgroundclose`` is | |
|
2057 | enabled. | |
|
2058 | (default: 384) | |
|
2059 | ||
|
2060 | ``backgroundclosethreadcount`` | |
|
2061 | Number of threads to process background file closes. Only relevant if | |
|
2062 | ``backgroundclose`` is enabled. | |
|
2063 | (default: 4) |
@@ -7,6 +7,8 b'' | |||
|
7 | 7 | |
|
8 | 8 | from __future__ import absolute_import |
|
9 | 9 | |
|
10 | import Queue | |
|
11 | import contextlib | |
|
10 | 12 | import errno |
|
11 | 13 | import glob |
|
12 | 14 | import os |
@@ -14,6 +16,7 b' import re' | |||
|
14 | 16 | import shutil |
|
15 | 17 | import stat |
|
16 | 18 | import tempfile |
|
19 | import threading | |
|
17 | 20 | |
|
18 | 21 | from .i18n import _ |
|
19 | 22 | from .node import wdirrev |
@@ -254,7 +257,7 b' class abstractvfs(object):' | |||
|
254 | 257 | return [] |
|
255 | 258 | |
|
256 | 259 | def open(self, path, mode="r", text=False, atomictemp=False, |
|
257 | notindexed=False): | |
|
260 | notindexed=False, backgroundclose=False): | |
|
258 | 261 | '''Open ``path`` file, which is relative to vfs root. |
|
259 | 262 | |
|
260 | 263 | Newly created directories are marked as "not to be indexed by |
@@ -262,7 +265,8 b' class abstractvfs(object):' | |||
|
262 | 265 | for "write" mode access. |
|
263 | 266 | ''' |
|
264 | 267 | self.open = self.__call__ |
|
265 |
return self.__call__(path, mode, text, atomictemp, notindexed |
|
|
268 | return self.__call__(path, mode, text, atomictemp, notindexed, | |
|
269 | backgroundclose=backgroundclose) | |
|
266 | 270 | |
|
267 | 271 | def read(self, path): |
|
268 | 272 | with self(path, 'rb') as fp: |
@@ -436,6 +440,27 b' class abstractvfs(object):' | |||
|
436 | 440 | for dirpath, dirs, files in os.walk(self.join(path), onerror=onerror): |
|
437 | 441 | yield (dirpath[prefixlen:], dirs, files) |
|
438 | 442 | |
|
443 | @contextlib.contextmanager | |
|
444 | def backgroundclosing(self, ui, expectedcount=-1): | |
|
445 | """Allow files to be closed asynchronously. | |
|
446 | ||
|
447 | When this context manager is active, ``backgroundclose`` can be passed | |
|
448 | to ``__call__``/``open`` to result in the file possibly being closed | |
|
449 | asynchronously, on a background thread. | |
|
450 | """ | |
|
451 | # This is an arbitrary restriction and could be changed if we ever | |
|
452 | # have a use case. | |
|
453 | vfs = getattr(self, 'vfs', self) | |
|
454 | if getattr(vfs, '_backgroundfilecloser', None): | |
|
455 | raise error.Abort('can only have 1 active background file closer') | |
|
456 | ||
|
457 | with backgroundfilecloser(ui, expectedcount=expectedcount) as bfc: | |
|
458 | try: | |
|
459 | vfs._backgroundfilecloser = bfc | |
|
460 | yield bfc | |
|
461 | finally: | |
|
462 | vfs._backgroundfilecloser = None | |
|
463 | ||
|
439 | 464 | class vfs(abstractvfs): |
|
440 | 465 | '''Operate files relative to a base directory |
|
441 | 466 | |
@@ -478,12 +503,25 b' class vfs(abstractvfs):' | |||
|
478 | 503 | os.chmod(name, self.createmode & 0o666) |
|
479 | 504 | |
|
480 | 505 | def __call__(self, path, mode="r", text=False, atomictemp=False, |
|
481 | notindexed=False): | |
|
506 | notindexed=False, backgroundclose=False): | |
|
482 | 507 | '''Open ``path`` file, which is relative to vfs root. |
|
483 | 508 | |
|
484 | 509 | Newly created directories are marked as "not to be indexed by |
|
485 | 510 | the content indexing service", if ``notindexed`` is specified |
|
486 | 511 | for "write" mode access. |
|
512 | ||
|
513 | If ``backgroundclose`` is passed, the file may be closed asynchronously. | |
|
514 | It can only be used if the ``self.backgroundclosing()`` context manager | |
|
515 | is active. This should only be specified if the following criteria hold: | |
|
516 | ||
|
517 | 1. There is a potential for writing thousands of files. Unless you | |
|
518 | are writing thousands of files, the performance benefits of | |
|
519 | asynchronously closing files is not realized. | |
|
520 | 2. Files are opened exactly once for the ``backgroundclosing`` | |
|
521 | active duration and are therefore free of race conditions between | |
|
522 | closing a file on a background thread and reopening it. (If the | |
|
523 | file were opened multiple times, there could be unflushed data | |
|
524 | because the original file handle hasn't been flushed/closed yet.) | |
|
487 | 525 | ''' |
|
488 | 526 | if self._audit: |
|
489 | 527 | r = util.checkosfilename(path) |
@@ -528,6 +566,14 b' class vfs(abstractvfs):' | |||
|
528 | 566 | fp = util.posixfile(f, mode) |
|
529 | 567 | if nlink == 0: |
|
530 | 568 | self._fixfilemode(f) |
|
569 | ||
|
570 | if backgroundclose: | |
|
571 | if not self._backgroundfilecloser: | |
|
572 | raise error.Abort('backgroundclose can only be used when a ' | |
|
573 | 'backgroundclosing context manager is active') | |
|
574 | ||
|
575 | fp = delayclosedfile(fp, self._backgroundfilecloser) | |
|
576 | ||
|
531 | 577 | return fp |
|
532 | 578 | |
|
533 | 579 | def symlink(self, src, dst): |
@@ -1211,3 +1257,123 b' def gddeltaconfig(ui):' | |||
|
1211 | 1257 | """ |
|
1212 | 1258 | # experimental config: format.generaldelta |
|
1213 | 1259 | return ui.configbool('format', 'generaldelta', False) |
|
1260 | ||
|
1261 | class delayclosedfile(object): | |
|
1262 | """Proxy for a file object whose close is delayed. | |
|
1263 | ||
|
1264 | Do not instantiate outside of the vfs layer. | |
|
1265 | """ | |
|
1266 | ||
|
1267 | def __init__(self, fh, closer): | |
|
1268 | object.__setattr__(self, '_origfh', fh) | |
|
1269 | object.__setattr__(self, '_closer', closer) | |
|
1270 | ||
|
1271 | def __getattr__(self, attr): | |
|
1272 | return getattr(self._origfh, attr) | |
|
1273 | ||
|
1274 | def __setattr__(self, attr, value): | |
|
1275 | return setattr(self._origfh, attr, value) | |
|
1276 | ||
|
1277 | def __delattr__(self, attr): | |
|
1278 | return delattr(self._origfh, attr) | |
|
1279 | ||
|
1280 | def __enter__(self): | |
|
1281 | return self._origfh.__enter__() | |
|
1282 | ||
|
1283 | def __exit__(self, exc_type, exc_value, exc_tb): | |
|
1284 | self._closer.close(self._origfh) | |
|
1285 | ||
|
1286 | def close(self): | |
|
1287 | self._closer.close(self._origfh) | |
|
1288 | ||
|
1289 | class backgroundfilecloser(object): | |
|
1290 | """Coordinates background closing of file handles on multiple threads.""" | |
|
1291 | def __init__(self, ui, expectedcount=-1): | |
|
1292 | self._running = False | |
|
1293 | self._entered = False | |
|
1294 | self._threads = [] | |
|
1295 | self._threadexception = None | |
|
1296 | ||
|
1297 | # Only Windows/NTFS has slow file closing. So only enable by default | |
|
1298 | # on that platform. But allow to be enabled elsewhere for testing. | |
|
1299 | defaultenabled = os.name == 'nt' | |
|
1300 | enabled = ui.configbool('worker', 'backgroundclose', defaultenabled) | |
|
1301 | ||
|
1302 | if not enabled: | |
|
1303 | return | |
|
1304 | ||
|
1305 | # There is overhead to starting and stopping the background threads. | |
|
1306 | # Don't do background processing unless the file count is large enough | |
|
1307 | # to justify it. | |
|
1308 | minfilecount = ui.configint('worker', 'backgroundcloseminfilecount', | |
|
1309 | 2048) | |
|
1310 | # FUTURE dynamically start background threads after minfilecount closes. | |
|
1311 | # (We don't currently have any callers that don't know their file count) | |
|
1312 | if expectedcount > 0 and expectedcount < minfilecount: | |
|
1313 | return | |
|
1314 | ||
|
1315 | # Windows defaults to a limit of 512 open files. A buffer of 128 | |
|
1316 | # should give us enough headway. | |
|
1317 | maxqueue = ui.configint('worker', 'backgroundclosemaxqueue', 384) | |
|
1318 | threadcount = ui.configint('worker', 'backgroundclosethreadcount', 4) | |
|
1319 | ||
|
1320 | ui.debug('starting %d threads for background file closing\n' % | |
|
1321 | threadcount) | |
|
1322 | ||
|
1323 | self._queue = Queue.Queue(maxsize=maxqueue) | |
|
1324 | self._running = True | |
|
1325 | ||
|
1326 | for i in range(threadcount): | |
|
1327 | t = threading.Thread(target=self._worker, name='backgroundcloser') | |
|
1328 | self._threads.append(t) | |
|
1329 | t.start() | |
|
1330 | ||
|
1331 | def __enter__(self): | |
|
1332 | self._entered = True | |
|
1333 | return self | |
|
1334 | ||
|
1335 | def __exit__(self, exc_type, exc_value, exc_tb): | |
|
1336 | self._running = False | |
|
1337 | ||
|
1338 | # Wait for threads to finish closing so open files don't linger for | |
|
1339 | # longer than lifetime of context manager. | |
|
1340 | for t in self._threads: | |
|
1341 | t.join() | |
|
1342 | ||
|
1343 | def _worker(self): | |
|
1344 | """Main routine for worker thread.""" | |
|
1345 | while True: | |
|
1346 | try: | |
|
1347 | fh = self._queue.get(block=True, timeout=0.100) | |
|
1348 | # Need to catch or the thread will terminate and | |
|
1349 | # we could orphan file descriptors. | |
|
1350 | try: | |
|
1351 | fh.close() | |
|
1352 | except Exception as e: | |
|
1353 | # Stash so can re-raise from main thread later. | |
|
1354 | self._threadexception = e | |
|
1355 | except Queue.Empty: | |
|
1356 | if not self._running: | |
|
1357 | break | |
|
1358 | ||
|
1359 | def close(self, fh): | |
|
1360 | """Schedule a file for closing.""" | |
|
1361 | if not self._entered: | |
|
1362 | raise error.Abort('can only call close() when context manager ' | |
|
1363 | 'active') | |
|
1364 | ||
|
1365 | # If a background thread encountered an exception, raise now so we fail | |
|
1366 | # fast. Otherwise we may potentially go on for minutes until the error | |
|
1367 | # is acted on. | |
|
1368 | if self._threadexception: | |
|
1369 | e = self._threadexception | |
|
1370 | self._threadexception = None | |
|
1371 | raise e | |
|
1372 | ||
|
1373 | # If we're not actively running, close synchronously. | |
|
1374 | if not self._running: | |
|
1375 | fh.close() | |
|
1376 | return | |
|
1377 | ||
|
1378 | self._queue.put(fh, block=True, timeout=None) | |
|
1379 |
General Comments 0
You need to be logged in to leave comments.
Login now