File indexing completed on 2025-01-18 10:17:50
0001 """
0002 This module provides helpers for C++11+ projects using pybind11.
0003
0004 LICENSE:
0005
0006 Copyright (c) 2016 Wenzel Jakob <wenzel.jakob@epfl.ch>, All rights reserved.
0007
0008 Redistribution and use in source and binary forms, with or without
0009 modification, are permitted provided that the following conditions are met:
0010
0011 1. Redistributions of source code must retain the above copyright notice, this
0012 list of conditions and the following disclaimer.
0013
0014 2. Redistributions in binary form must reproduce the above copyright notice,
0015 this list of conditions and the following disclaimer in the documentation
0016 and/or other materials provided with the distribution.
0017
0018 3. Neither the name of the copyright holder nor the names of its contributors
0019 may be used to endorse or promote products derived from this software
0020 without specific prior written permission.
0021
0022 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
0023 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
0024 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
0025 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
0026 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
0027 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
0028 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
0029 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
0030 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
0031 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
0032 """
0033
0034
0035
0036
0037
0038
0039
0040 import contextlib
0041 import os
0042 import platform
0043 import shlex
0044 import shutil
0045 import sys
0046 import sysconfig
0047 import tempfile
0048 import threading
0049 import warnings
0050 from functools import lru_cache
0051 from pathlib import Path
0052 from typing import (
0053 Any,
0054 Callable,
0055 Dict,
0056 Iterable,
0057 Iterator,
0058 List,
0059 Optional,
0060 Tuple,
0061 TypeVar,
0062 Union,
0063 )
0064
0065 try:
0066 from setuptools import Extension as _Extension
0067 from setuptools.command.build_ext import build_ext as _build_ext
0068 except ImportError:
0069 from distutils.command.build_ext import build_ext as _build_ext
0070 from distutils.extension import Extension as _Extension
0071
0072 import distutils.ccompiler
0073 import distutils.errors
0074
0075 WIN = sys.platform.startswith("win32") and "mingw" not in sysconfig.get_platform()
0076 MACOS = sys.platform.startswith("darwin")
0077 STD_TMPL = "/std:c++{}" if WIN else "-std=c++{}"
0078
0079
0080
0081
0082
0083
0084
0085
0086
0087 class Pybind11Extension(_Extension):
0088 """
0089 Build a C++11+ Extension module with pybind11. This automatically adds the
0090 recommended flags when you init the extension and assumes C++ sources - you
0091 can further modify the options yourself.
0092
0093 The customizations are:
0094
0095 * ``/EHsc`` and ``/bigobj`` on Windows
0096 * ``stdlib=libc++`` on macOS
0097 * ``visibility=hidden`` and ``-g0`` on Unix
0098
0099 Finally, you can set ``cxx_std`` via constructor or afterwards to enable
0100 flags for C++ std, and a few extra helper flags related to the C++ standard
0101 level. It is _highly_ recommended you either set this, or use the provided
0102 ``build_ext``, which will search for the highest supported extension for
0103 you if the ``cxx_std`` property is not set. Do not set the ``cxx_std``
0104 property more than once, as flags are added when you set it. Set the
0105 property to None to disable the addition of C++ standard flags.
0106
0107 If you want to add pybind11 headers manually, for example for an exact
0108 git checkout, then set ``include_pybind11=False``.
0109 """
0110
0111
0112
0113
0114 def _add_cflags(self, flags: List[str]) -> None:
0115 self.extra_compile_args[:0] = flags
0116
0117 def _add_ldflags(self, flags: List[str]) -> None:
0118 self.extra_link_args[:0] = flags
0119
0120 def __init__(self, *args: Any, **kwargs: Any) -> None:
0121
0122 self._cxx_level = 0
0123 cxx_std = kwargs.pop("cxx_std", 0)
0124
0125 if "language" not in kwargs:
0126 kwargs["language"] = "c++"
0127
0128 include_pybind11 = kwargs.pop("include_pybind11", True)
0129
0130 super().__init__(*args, **kwargs)
0131
0132
0133 if include_pybind11:
0134
0135 try:
0136 import pybind11
0137
0138 pyinc = pybind11.get_include()
0139
0140 if pyinc not in self.include_dirs:
0141 self.include_dirs.append(pyinc)
0142 except ModuleNotFoundError:
0143 pass
0144
0145 self.cxx_std = cxx_std
0146
0147 cflags = []
0148 ldflags = []
0149 if WIN:
0150 cflags += ["/EHsc", "/bigobj"]
0151 else:
0152 cflags += ["-fvisibility=hidden"]
0153 env_cflags = os.environ.get("CFLAGS", "")
0154 env_cppflags = os.environ.get("CPPFLAGS", "")
0155 c_cpp_flags = shlex.split(env_cflags) + shlex.split(env_cppflags)
0156 if not any(opt.startswith("-g") for opt in c_cpp_flags):
0157 cflags += ["-g0"]
0158 if MACOS:
0159 cflags += ["-stdlib=libc++"]
0160 ldflags += ["-stdlib=libc++"]
0161 self._add_cflags(cflags)
0162 self._add_ldflags(ldflags)
0163
0164 @property
0165 def cxx_std(self) -> int:
0166 """
0167 The CXX standard level. If set, will add the required flags. If left at
0168 0, it will trigger an automatic search when pybind11's build_ext is
0169 used. If None, will have no effect. Besides just the flags, this may
0170 add a macos-min 10.9 or 10.14 flag if MACOSX_DEPLOYMENT_TARGET is
0171 unset.
0172 """
0173 return self._cxx_level
0174
0175 @cxx_std.setter
0176 def cxx_std(self, level: int) -> None:
0177
0178 if self._cxx_level:
0179 warnings.warn("You cannot safely change the cxx_level after setting it!")
0180
0181
0182
0183 if WIN and level == 11:
0184 level = 14
0185
0186 self._cxx_level = level
0187
0188 if not level:
0189 return
0190
0191 cflags = [STD_TMPL.format(level)]
0192 ldflags = []
0193
0194 if MACOS and "MACOSX_DEPLOYMENT_TARGET" not in os.environ:
0195
0196
0197
0198
0199
0200 current_macos = tuple(int(x) for x in platform.mac_ver()[0].split(".")[:2])
0201 desired_macos = (10, 9) if level < 17 else (10, 14)
0202 macos_string = ".".join(str(x) for x in min(current_macos, desired_macos))
0203 macosx_min = f"-mmacosx-version-min={macos_string}"
0204 cflags += [macosx_min]
0205 ldflags += [macosx_min]
0206
0207 self._add_cflags(cflags)
0208 self._add_ldflags(ldflags)
0209
0210
0211
0212 tmp_chdir_lock = threading.Lock()
0213
0214
0215 @contextlib.contextmanager
0216 def tmp_chdir() -> Iterator[str]:
0217 "Prepare and enter a temporary directory, cleanup when done"
0218
0219
0220 with tmp_chdir_lock:
0221 olddir = os.getcwd()
0222 try:
0223 tmpdir = tempfile.mkdtemp()
0224 os.chdir(tmpdir)
0225 yield tmpdir
0226 finally:
0227 os.chdir(olddir)
0228 shutil.rmtree(tmpdir)
0229
0230
0231
0232 def has_flag(compiler: Any, flag: str) -> bool:
0233 """
0234 Return the flag if a flag name is supported on the
0235 specified compiler, otherwise None (can be used as a boolean).
0236 If multiple flags are passed, return the first that matches.
0237 """
0238
0239 with tmp_chdir():
0240 fname = Path("flagcheck.cpp")
0241
0242 fname.write_text("int main (int, char **) { return 0; }", encoding="utf-8")
0243
0244 try:
0245 compiler.compile([str(fname)], extra_postargs=[flag])
0246 except distutils.errors.CompileError:
0247 return False
0248 return True
0249
0250
0251
0252 cpp_flag_cache = None
0253
0254
0255 @lru_cache()
0256 def auto_cpp_level(compiler: Any) -> Union[str, int]:
0257 """
0258 Return the max supported C++ std level (17, 14, or 11). Returns latest on Windows.
0259 """
0260
0261 if WIN:
0262 return "latest"
0263
0264 levels = [17, 14, 11]
0265
0266 for level in levels:
0267 if has_flag(compiler, STD_TMPL.format(level)):
0268 return level
0269
0270 msg = "Unsupported compiler -- at least C++11 support is needed!"
0271 raise RuntimeError(msg)
0272
0273
0274 class build_ext(_build_ext):
0275 """
0276 Customized build_ext that allows an auto-search for the highest supported
0277 C++ level for Pybind11Extension. This is only needed for the auto-search
0278 for now, and is completely optional otherwise.
0279 """
0280
0281 def build_extensions(self) -> None:
0282 """
0283 Build extensions, injecting C++ std for Pybind11Extension if needed.
0284 """
0285
0286 for ext in self.extensions:
0287 if hasattr(ext, "_cxx_level") and ext._cxx_level == 0:
0288 ext.cxx_std = auto_cpp_level(self.compiler)
0289
0290 super().build_extensions()
0291
0292
0293 def intree_extensions(
0294 paths: Iterable[str], package_dir: Optional[Dict[str, str]] = None
0295 ) -> List[Pybind11Extension]:
0296 """
0297 Generate Pybind11Extensions from source files directly located in a Python
0298 source tree.
0299
0300 ``package_dir`` behaves as in ``setuptools.setup``. If unset, the Python
0301 package root parent is determined as the first parent directory that does
0302 not contain an ``__init__.py`` file.
0303 """
0304 exts = []
0305
0306 if package_dir is None:
0307 for path in paths:
0308 parent, _ = os.path.split(path)
0309 while os.path.exists(os.path.join(parent, "__init__.py")):
0310 parent, _ = os.path.split(parent)
0311 relname, _ = os.path.splitext(os.path.relpath(path, parent))
0312 qualified_name = relname.replace(os.path.sep, ".")
0313 exts.append(Pybind11Extension(qualified_name, [path]))
0314 return exts
0315
0316 for path in paths:
0317 for prefix, parent in package_dir.items():
0318 if path.startswith(parent):
0319 relname, _ = os.path.splitext(os.path.relpath(path, parent))
0320 qualified_name = relname.replace(os.path.sep, ".")
0321 if prefix:
0322 qualified_name = prefix + "." + qualified_name
0323 exts.append(Pybind11Extension(qualified_name, [path]))
0324 break
0325 else:
0326 msg = (
0327 f"path {path} is not a child of any of the directories listed "
0328 f"in 'package_dir' ({package_dir})"
0329 )
0330 raise ValueError(msg)
0331
0332 return exts
0333
0334
0335 def naive_recompile(obj: str, src: str) -> bool:
0336 """
0337 This will recompile only if the source file changes. It does not check
0338 header files, so a more advanced function or Ccache is better if you have
0339 editable header files in your package.
0340 """
0341 return os.stat(obj).st_mtime < os.stat(src).st_mtime
0342
0343
0344 def no_recompile(obg: str, src: str) -> bool:
0345 """
0346 This is the safest but slowest choice (and is the default) - will always
0347 recompile sources.
0348 """
0349 return True
0350
0351
0352 S = TypeVar("S", bound="ParallelCompile")
0353
0354 CCompilerMethod = Callable[
0355 [
0356 distutils.ccompiler.CCompiler,
0357 List[str],
0358 Optional[str],
0359 Optional[Union[Tuple[str], Tuple[str, Optional[str]]]],
0360 Optional[List[str]],
0361 bool,
0362 Optional[List[str]],
0363 Optional[List[str]],
0364 Optional[List[str]],
0365 ],
0366 List[str],
0367 ]
0368
0369
0370
0371
0372
0373
0374
0375 class ParallelCompile:
0376 """
0377 Make a parallel compile function. Inspired by
0378 numpy.distutils.ccompiler.CCompiler.compile and cppimport.
0379
0380 This takes several arguments that allow you to customize the compile
0381 function created:
0382
0383 envvar:
0384 Set an environment variable to control the compilation threads, like
0385 NPY_NUM_BUILD_JOBS
0386 default:
0387 0 will automatically multithread, or 1 will only multithread if the
0388 envvar is set.
0389 max:
0390 The limit for automatic multithreading if non-zero
0391 needs_recompile:
0392 A function of (obj, src) that returns True when recompile is needed. No
0393 effect in isolated mode; use ccache instead, see
0394 https://github.com/matplotlib/matplotlib/issues/1507/
0395
0396 To use::
0397
0398 ParallelCompile("NPY_NUM_BUILD_JOBS").install()
0399
0400 or::
0401
0402 with ParallelCompile("NPY_NUM_BUILD_JOBS"):
0403 setup(...)
0404
0405 By default, this assumes all files need to be recompiled. A smarter
0406 function can be provided via needs_recompile. If the output has not yet
0407 been generated, the compile will always run, and this function is not
0408 called.
0409 """
0410
0411 __slots__ = ("envvar", "default", "max", "_old", "needs_recompile")
0412
0413 def __init__(
0414 self,
0415 envvar: Optional[str] = None,
0416 default: int = 0,
0417 max: int = 0,
0418 needs_recompile: Callable[[str, str], bool] = no_recompile,
0419 ) -> None:
0420 self.envvar = envvar
0421 self.default = default
0422 self.max = max
0423 self.needs_recompile = needs_recompile
0424 self._old: List[CCompilerMethod] = []
0425
0426 def function(self) -> CCompilerMethod:
0427 """
0428 Builds a function object usable as distutils.ccompiler.CCompiler.compile.
0429 """
0430
0431 def compile_function(
0432 compiler: distutils.ccompiler.CCompiler,
0433 sources: List[str],
0434 output_dir: Optional[str] = None,
0435 macros: Optional[Union[Tuple[str], Tuple[str, Optional[str]]]] = None,
0436 include_dirs: Optional[List[str]] = None,
0437 debug: bool = False,
0438 extra_preargs: Optional[List[str]] = None,
0439 extra_postargs: Optional[List[str]] = None,
0440 depends: Optional[List[str]] = None,
0441 ) -> Any:
0442
0443
0444 macros, objects, extra_postargs, pp_opts, build = compiler._setup_compile(
0445 output_dir, macros, include_dirs, sources, depends, extra_postargs
0446 )
0447 cc_args = compiler._get_cc_args(pp_opts, debug, extra_preargs)
0448
0449
0450 threads = self.default
0451
0452
0453 if self.envvar is not None:
0454 threads = int(os.environ.get(self.envvar, self.default))
0455
0456 def _single_compile(obj: Any) -> None:
0457 try:
0458 src, ext = build[obj]
0459 except KeyError:
0460 return
0461
0462 if not os.path.exists(obj) or self.needs_recompile(obj, src):
0463 compiler._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
0464
0465 try:
0466
0467
0468 import multiprocessing.synchronize
0469 from multiprocessing.pool import ThreadPool
0470 except ImportError:
0471 threads = 1
0472
0473 if threads == 0:
0474 try:
0475 threads = multiprocessing.cpu_count()
0476 threads = self.max if self.max and self.max < threads else threads
0477 except NotImplementedError:
0478 threads = 1
0479
0480 if threads > 1:
0481 with ThreadPool(threads) as pool:
0482 for _ in pool.imap_unordered(_single_compile, objects):
0483 pass
0484 else:
0485 for ob in objects:
0486 _single_compile(ob)
0487
0488 return objects
0489
0490 return compile_function
0491
0492 def install(self: S) -> S:
0493 """
0494 Installs the compile function into distutils.ccompiler.CCompiler.compile.
0495 """
0496 distutils.ccompiler.CCompiler.compile = self.function()
0497 return self
0498
0499 def __enter__(self: S) -> S:
0500 self._old.append(distutils.ccompiler.CCompiler.compile)
0501 return self.install()
0502
0503 def __exit__(self, *args: Any) -> None:
0504 distutils.ccompiler.CCompiler.compile = self._old.pop()