##// END OF EJS Templates
copies: split the combination of the copies mapping in its own function...
copies: split the combination of the copies mapping in its own function In some case, this part take up to 95% of the copy tracing that take about a hundred second. This poor performance comes from the fact we keep duplciating and merging dictionary that are mostly similar. I want to experiment with smarter native code to do this, so I need to isolate the function first.

File last commit:

r43347:687b865b default
r44178:0cc91600 default
Show More
osutil.py
115 lines | 3.8 KiB | text/x-python | PythonLexer
Yuya Nishihara
cffi: split modules from pure...
r32512 # osutil.py - CFFI version of osutil.c
#
# Copyright 2016 Maciej Fijalkowski <fijall@gmail.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import
import os
import stat as statmod
from ..pure.osutil import *
Augie Fackler
formatting: blacken the codebase...
r43346 from .. import pycompat
Yuya Nishihara
cffi: split modules from pure...
r32512
Jun Wu
codemod: use pycompat.isdarwin...
r34648 if pycompat.isdarwin:
Yuya Nishihara
cffi: split modules from pure...
r32512 from . import _osutil
ffi = _osutil.ffi
lib = _osutil.lib
listdir_batch_size = 4096
# tweakable number, only affects performance, which chunks
# of bytes do we get back from getattrlistbulk
Augie Fackler
formatting: blacken the codebase...
r43346 attrkinds = [None] * 20 # we need the max no for enum VXXX, 20 is plenty
Yuya Nishihara
cffi: split modules from pure...
r32512
attrkinds[lib.VREG] = statmod.S_IFREG
attrkinds[lib.VDIR] = statmod.S_IFDIR
attrkinds[lib.VLNK] = statmod.S_IFLNK
attrkinds[lib.VBLK] = statmod.S_IFBLK
attrkinds[lib.VCHR] = statmod.S_IFCHR
attrkinds[lib.VFIFO] = statmod.S_IFIFO
attrkinds[lib.VSOCK] = statmod.S_IFSOCK
class stat_res(object):
def __init__(self, st_mode, st_mtime, st_size):
self.st_mode = st_mode
self.st_mtime = st_mtime
self.st_size = st_size
Augie Fackler
formatting: byteify all mercurial/ and hgext/ string literals...
r43347 tv_sec_ofs = ffi.offsetof(b"struct timespec", b"tv_sec")
buf = ffi.new(b"char[]", listdir_batch_size)
Yuya Nishihara
cffi: split modules from pure...
r32512
def listdirinternal(dfd, req, stat, skip):
ret = []
while True:
r = lib.getattrlistbulk(dfd, req, buf, listdir_batch_size, 0)
if r == 0:
break
if r == -1:
raise OSError(ffi.errno, os.strerror(ffi.errno))
Augie Fackler
formatting: byteify all mercurial/ and hgext/ string literals...
r43347 cur = ffi.cast(b"val_attrs_t*", buf)
Yuya Nishihara
cffi: split modules from pure...
r32512 for i in range(r):
lgt = cur.length
Augie Fackler
formatting: byteify all mercurial/ and hgext/ string literals...
r43347 assert lgt == ffi.cast(b'uint32_t*', cur)[0]
Yuya Nishihara
cffi: split modules from pure...
r32512 ofs = cur.name_info.attr_dataoffset
str_lgt = cur.name_info.attr_length
Augie Fackler
formatting: byteify all mercurial/ and hgext/ string literals...
r43347 base_ofs = ffi.offsetof(b'val_attrs_t', b'name_info')
Augie Fackler
formatting: blacken the codebase...
r43346 name = str(
ffi.buffer(
Augie Fackler
formatting: byteify all mercurial/ and hgext/ string literals...
r43347 ffi.cast(b"char*", cur) + base_ofs + ofs, str_lgt - 1
Augie Fackler
formatting: blacken the codebase...
r43346 )
)
Yuya Nishihara
cffi: split modules from pure...
r32512 tp = attrkinds[cur.obj_type]
Augie Fackler
formatting: byteify all mercurial/ and hgext/ string literals...
r43347 if name == b"." or name == b"..":
Yuya Nishihara
cffi: split modules from pure...
r32512 continue
if skip == name and tp == statmod.S_ISDIR:
return []
if stat:
mtime = cur.mtime.tv_sec
Augie Fackler
formatting: blacken the codebase...
r43346 mode = (cur.accessmask & ~lib.S_IFMT) | tp
ret.append(
(
name,
tp,
stat_res(
st_mode=mode,
st_mtime=mtime,
st_size=cur.datalength,
),
)
)
Yuya Nishihara
cffi: split modules from pure...
r32512 else:
ret.append((name, tp))
Augie Fackler
formatting: blacken the codebase...
r43346 cur = ffi.cast(
Augie Fackler
formatting: byteify all mercurial/ and hgext/ string literals...
r43347 b"val_attrs_t*", int(ffi.cast(b"intptr_t", cur)) + lgt
Augie Fackler
formatting: blacken the codebase...
r43346 )
Yuya Nishihara
cffi: split modules from pure...
r32512 return ret
def listdir(path, stat=False, skip=None):
Augie Fackler
formatting: byteify all mercurial/ and hgext/ string literals...
r43347 req = ffi.new(b"struct attrlist*")
Yuya Nishihara
cffi: split modules from pure...
r32512 req.bitmapcount = lib.ATTR_BIT_MAP_COUNT
Augie Fackler
formatting: blacken the codebase...
r43346 req.commonattr = (
lib.ATTR_CMN_RETURNED_ATTRS
| lib.ATTR_CMN_NAME
| lib.ATTR_CMN_OBJTYPE
| lib.ATTR_CMN_ACCESSMASK
| lib.ATTR_CMN_MODTIME
)
Yuya Nishihara
cffi: split modules from pure...
r32512 req.fileattr = lib.ATTR_FILE_DATALENGTH
dfd = lib.open(path, lib.O_RDONLY, 0)
if dfd == -1:
raise OSError(ffi.errno, os.strerror(ffi.errno))
try:
ret = listdirinternal(dfd, req, stat, skip)
finally:
try:
lib.close(dfd)
except BaseException:
Augie Fackler
formatting: blacken the codebase...
r43346 pass # we ignore all the errors from closing, not
Yuya Nishihara
cffi: split modules from pure...
r32512 # much we can do about that
return ret