make_cffi.py
187 lines
| 5.6 KiB
| text/x-python
|
PythonLexer
|
r30435 | # Copyright (c) 2016-present, Gregory Szorc | ||
# All rights reserved. | ||||
# | ||||
# This software may be modified and distributed under the terms | ||||
# of the BSD license. See the LICENSE file for details. | ||||
from __future__ import absolute_import | ||||
import cffi | ||||
|
r30822 | import distutils.ccompiler | ||
|
r30435 | import os | ||
|
r30895 | import re | ||
|
r30822 | import subprocess | ||
import tempfile | ||||
|
r30435 | |||
HERE = os.path.abspath(os.path.dirname(__file__)) | ||||
SOURCES = ['zstd/%s' % p for p in ( | ||||
'common/entropy_common.c', | ||||
'common/error_private.c', | ||||
'common/fse_decompress.c', | ||||
|
r30895 | 'common/pool.c', | ||
'common/threading.c', | ||||
|
r30435 | 'common/xxhash.c', | ||
'common/zstd_common.c', | ||||
'compress/fse_compress.c', | ||||
'compress/huf_compress.c', | ||||
'compress/zstd_compress.c', | ||||
|
r31796 | 'compress/zstdmt_compress.c', | ||
|
r30435 | 'decompress/huf_decompress.c', | ||
'decompress/zstd_decompress.c', | ||||
|
r30895 | 'dictBuilder/cover.c', | ||
|
r30435 | 'dictBuilder/divsufsort.c', | ||
'dictBuilder/zdict.c', | ||||
)] | ||||
|
r31796 | # Headers whose preprocessed output will be fed into cdef(). | ||
|
r30895 | HEADERS = [os.path.join(HERE, 'zstd', *p) for p in ( | ||
('zstd.h',), | ||||
|
r31796 | ('compress', 'zstdmt_compress.h'), | ||
|
r30895 | ('dictBuilder', 'zdict.h'), | ||
)] | ||||
|
r30435 | INCLUDE_DIRS = [os.path.join(HERE, d) for d in ( | ||
'zstd', | ||||
'zstd/common', | ||||
'zstd/compress', | ||||
'zstd/decompress', | ||||
'zstd/dictBuilder', | ||||
)] | ||||
|
r30822 | # cffi can't parse some of the primitives in zstd.h. So we invoke the | ||
# preprocessor and feed its output into cffi. | ||||
compiler = distutils.ccompiler.new_compiler() | ||||
# Needed for MSVC. | ||||
if hasattr(compiler, 'initialize'): | ||||
compiler.initialize() | ||||
# Distutils doesn't set compiler.preprocessor, so invoke the preprocessor | ||||
# manually. | ||||
if compiler.compiler_type == 'unix': | ||||
args = list(compiler.executables['compiler']) | ||||
args.extend([ | ||||
'-E', | ||||
'-DZSTD_STATIC_LINKING_ONLY', | ||||
|
r30895 | '-DZDICT_STATIC_LINKING_ONLY', | ||
|
r30822 | ]) | ||
elif compiler.compiler_type == 'msvc': | ||||
args = [compiler.cc] | ||||
args.extend([ | ||||
'/EP', | ||||
'/DZSTD_STATIC_LINKING_ONLY', | ||||
|
r30895 | '/DZDICT_STATIC_LINKING_ONLY', | ||
|
r30822 | ]) | ||
else: | ||||
raise Exception('unsupported compiler type: %s' % compiler.compiler_type) | ||||
|
r30895 | def preprocess(path): | ||
with open(path, 'rb') as fh: | ||||
|
r31796 | lines = [] | ||
for l in fh: | ||||
# zstd.h includes <stddef.h>, which is also included by cffi's | ||||
# boilerplate. This can lead to duplicate declarations. So we strip | ||||
# this include from the preprocessor invocation. | ||||
# | ||||
# The same things happens for including zstd.h, so give it the same | ||||
# treatment. | ||||
# | ||||
# We define ZSTD_STATIC_LINKING_ONLY, which is redundant with the inline | ||||
# #define in zstdmt_compress.h and results in a compiler warning. So drop | ||||
# the inline #define. | ||||
if l.startswith((b'#include <stddef.h>', | ||||
b'#include "zstd.h"', | ||||
b'#define ZSTD_STATIC_LINKING_ONLY')): | ||||
continue | ||||
# ZSTDLIB_API may not be defined if we dropped zstd.h. It isn't | ||||
# important so just filter it out. | ||||
if l.startswith(b'ZSTDLIB_API'): | ||||
l = l[len(b'ZSTDLIB_API '):] | ||||
lines.append(l) | ||||
|
r30822 | |||
|
r30895 | fd, input_file = tempfile.mkstemp(suffix='.h') | ||
os.write(fd, b''.join(lines)) | ||||
os.close(fd) | ||||
|
r30822 | |||
|
r30895 | try: | ||
process = subprocess.Popen(args + [input_file], stdout=subprocess.PIPE) | ||||
output = process.communicate()[0] | ||||
ret = process.poll() | ||||
if ret: | ||||
raise Exception('preprocessor exited with error') | ||||
|
r30822 | |||
|
r30895 | return output | ||
finally: | ||||
os.unlink(input_file) | ||||
|
r30822 | |||
|
r30895 | |||
def normalize_output(output): | ||||
|
r30822 | lines = [] | ||
for line in output.splitlines(): | ||||
# CFFI's parser doesn't like __attribute__ on UNIX compilers. | ||||
if line.startswith(b'__attribute__ ((visibility ("default"))) '): | ||||
line = line[len(b'__attribute__ ((visibility ("default"))) '):] | ||||
|
r30895 | if line.startswith(b'__attribute__((deprecated('): | ||
continue | ||||
elif b'__declspec(deprecated(' in line: | ||||
continue | ||||
|
r30822 | lines.append(line) | ||
return b'\n'.join(lines) | ||||
|
r30435 | |||
|
r30895 | |||
|
r30435 | ffi = cffi.FFI() | ||
|
r31796 | # *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning | ||
# when cffi uses the function. Since we statically link against zstd, even | ||||
# if we use the deprecated functions it shouldn't be a huge problem. | ||||
|
r30435 | ffi.set_source('_zstd_cffi', ''' | ||
|
r30895 | #include "mem.h" | ||
|
r30435 | #define ZSTD_STATIC_LINKING_ONLY | ||
#include "zstd.h" | ||||
|
r30895 | #define ZDICT_STATIC_LINKING_ONLY | ||
|
r31796 | #define ZDICT_DISABLE_DEPRECATE_WARNINGS | ||
|
r30895 | #include "zdict.h" | ||
|
r31796 | #include "zstdmt_compress.h" | ||
|
r30822 | ''', sources=SOURCES, include_dirs=INCLUDE_DIRS) | ||
|
r30435 | |||
|
r30895 | DEFINE = re.compile(b'^\\#define ([a-zA-Z0-9_]+) ') | ||
sources = [] | ||||
|
r31796 | # Feed normalized preprocessor output for headers into the cdef parser. | ||
|
r30895 | for header in HEADERS: | ||
preprocessed = preprocess(header) | ||||
sources.append(normalize_output(preprocessed)) | ||||
|
r31796 | # #define's are effectively erased as part of going through preprocessor. | ||
# So perform a manual pass to re-add those to the cdef source. | ||||
|
r30895 | with open(header, 'rb') as fh: | ||
for line in fh: | ||||
line = line.strip() | ||||
m = DEFINE.match(line) | ||||
if not m: | ||||
continue | ||||
|
r31796 | if m.group(1) == b'ZSTD_STATIC_LINKING_ONLY': | ||
continue | ||||
|
r30895 | # The parser doesn't like some constants with complex values. | ||
if m.group(1) in (b'ZSTD_LIB_VERSION', b'ZSTD_VERSION_STRING'): | ||||
continue | ||||
|
r31796 | # The ... is magic syntax by the cdef parser to resolve the | ||
# value at compile time. | ||||
|
r30895 | sources.append(m.group(0) + b' ...') | ||
|
r31796 | cdeflines = b'\n'.join(sources).splitlines() | ||
cdeflines = [l for l in cdeflines if l.strip()] | ||||
ffi.cdef(b'\n'.join(cdeflines).decode('latin1')) | ||||
|
r30435 | |||
if __name__ == '__main__': | ||||
ffi.compile() | ||||