# Copyright (c) 2016-present, Gregory Szorc # All rights reserved. # # This software may be modified and distributed under the terms # of the BSD license. See the LICENSE file for details. from __future__ import absolute_import import cffi import distutils.ccompiler import os import re import subprocess import tempfile HERE = os.path.abspath(os.path.dirname(__file__)) SOURCES = [ "zstd/%s" % p for p in ( "common/debug.c", "common/entropy_common.c", "common/error_private.c", "common/fse_decompress.c", "common/pool.c", "common/threading.c", "common/xxhash.c", "common/zstd_common.c", "compress/fse_compress.c", "compress/hist.c", "compress/huf_compress.c", "compress/zstd_compress.c", "compress/zstd_compress_literals.c", "compress/zstd_compress_sequences.c", "compress/zstd_double_fast.c", "compress/zstd_fast.c", "compress/zstd_lazy.c", "compress/zstd_ldm.c", "compress/zstd_opt.c", "compress/zstdmt_compress.c", "decompress/huf_decompress.c", "decompress/zstd_ddict.c", "decompress/zstd_decompress.c", "decompress/zstd_decompress_block.c", "dictBuilder/cover.c", "dictBuilder/fastcover.c", "dictBuilder/divsufsort.c", "dictBuilder/zdict.c", ) ] # Headers whose preprocessed output will be fed into cdef(). HEADERS = [ os.path.join(HERE, "zstd", *p) for p in (("zstd.h",), ("dictBuilder", "zdict.h"),) ] INCLUDE_DIRS = [ os.path.join(HERE, d) for d in ( "zstd", "zstd/common", "zstd/compress", "zstd/decompress", "zstd/dictBuilder", ) ] # cffi can't parse some of the primitives in zstd.h. So we invoke the # preprocessor and feed its output into cffi. compiler = distutils.ccompiler.new_compiler() # Needed for MSVC. if hasattr(compiler, "initialize"): compiler.initialize() # Distutils doesn't set compiler.preprocessor, so invoke the preprocessor # manually. if compiler.compiler_type == "unix": args = list(compiler.executables["compiler"]) args.extend( ["-E", "-DZSTD_STATIC_LINKING_ONLY", "-DZDICT_STATIC_LINKING_ONLY",] ) elif compiler.compiler_type == "msvc": args = [compiler.cc] args.extend( ["/EP", "/DZSTD_STATIC_LINKING_ONLY", "/DZDICT_STATIC_LINKING_ONLY",] ) else: raise Exception("unsupported compiler type: %s" % compiler.compiler_type) def preprocess(path): with open(path, "rb") as fh: lines = [] it = iter(fh) for l in it: # zstd.h includes , which is also included by cffi's # boilerplate. This can lead to duplicate declarations. So we strip # this include from the preprocessor invocation. # # The same things happens for including zstd.h, so give it the same # treatment. # # We define ZSTD_STATIC_LINKING_ONLY, which is redundant with the inline # #define in zstdmt_compress.h and results in a compiler warning. So drop # the inline #define. if l.startswith( ( b"#include ", b'#include "zstd.h"', b"#define ZSTD_STATIC_LINKING_ONLY", ) ): continue # The preprocessor environment on Windows doesn't define include # paths, so the #include of limits.h fails. We work around this # by removing that import and defining INT_MAX ourselves. This is # a bit hacky. But it gets the job done. # TODO make limits.h work on Windows so we ensure INT_MAX is # correct. if l.startswith(b"#include "): l = b"#define INT_MAX 2147483647\n" # ZSTDLIB_API may not be defined if we dropped zstd.h. It isn't # important so just filter it out. if l.startswith(b"ZSTDLIB_API"): l = l[len(b"ZSTDLIB_API ") :] lines.append(l) fd, input_file = tempfile.mkstemp(suffix=".h") os.write(fd, b"".join(lines)) os.close(fd) try: env = dict(os.environ) if getattr(compiler, "_paths", None): env["PATH"] = compiler._paths process = subprocess.Popen( args + [input_file], stdout=subprocess.PIPE, env=env ) output = process.communicate()[0] ret = process.poll() if ret: raise Exception("preprocessor exited with error") return output finally: os.unlink(input_file) def normalize_output(output): lines = [] for line in output.splitlines(): # CFFI's parser doesn't like __attribute__ on UNIX compilers. if line.startswith(b'__attribute__ ((visibility ("default"))) '): line = line[len(b'__attribute__ ((visibility ("default"))) ') :] if line.startswith(b"__attribute__((deprecated("): continue elif b"__declspec(deprecated(" in line: continue lines.append(line) return b"\n".join(lines) ffi = cffi.FFI() # zstd.h uses a possible undefined MIN(). Define it until # https://github.com/facebook/zstd/issues/976 is fixed. # *_DISABLE_DEPRECATE_WARNINGS prevents the compiler from emitting a warning # when cffi uses the function. Since we statically link against zstd, even # if we use the deprecated functions it shouldn't be a huge problem. ffi.set_source( "_zstd_cffi", """ #define MIN(a,b) ((a)<(b) ? (a) : (b)) #define ZSTD_STATIC_LINKING_ONLY #include #define ZDICT_STATIC_LINKING_ONLY #define ZDICT_DISABLE_DEPRECATE_WARNINGS #include """, sources=SOURCES, include_dirs=INCLUDE_DIRS, extra_compile_args=["-DZSTD_MULTITHREAD"], ) DEFINE = re.compile(b"^\\#define ([a-zA-Z0-9_]+) ") sources = [] # Feed normalized preprocessor output for headers into the cdef parser. for header in HEADERS: preprocessed = preprocess(header) sources.append(normalize_output(preprocessed)) # #define's are effectively erased as part of going through preprocessor. # So perform a manual pass to re-add those to the cdef source. with open(header, "rb") as fh: for line in fh: line = line.strip() m = DEFINE.match(line) if not m: continue if m.group(1) == b"ZSTD_STATIC_LINKING_ONLY": continue # The parser doesn't like some constants with complex values. if m.group(1) in (b"ZSTD_LIB_VERSION", b"ZSTD_VERSION_STRING"): continue # The ... is magic syntax by the cdef parser to resolve the # value at compile time. sources.append(m.group(0) + b" ...") cdeflines = b"\n".join(sources).splitlines() cdeflines = [l for l in cdeflines if l.strip()] ffi.cdef(b"\n".join(cdeflines).decode("latin1")) if __name__ == "__main__": ffi.compile()