include/pybind11/embed.h

0001 /*
0002     pybind11/embed.h: Support for embedding the interpreter
0003
0004     Copyright (c) 2017 Wenzel Jakob <wenzel.jakob@epfl.ch>
0005
0006     All rights reserved. Use of this source code is governed by a
0007     BSD-style license that can be found in the LICENSE file.
0008 */
0009
0010 #pragma once
0011
0012 #include "pybind11.h"
0013 #include "eval.h"
0014
0015 #include <memory>
0016 #include <vector>
0017
0018 #if defined(PYPY_VERSION)
0019 #    error Embedding the interpreter is not supported with PyPy
0020 #endif
0021
0022 #define PYBIND11_EMBEDDED_MODULE_IMPL(name)                                                       \
0023     extern "C" PyObject *pybind11_init_impl_##name();                                             \
0024     extern "C" PyObject *pybind11_init_impl_##name() { return pybind11_init_wrapper_##name(); }
0025
0026 /** \rst
0027     Add a new module to the table of builtins for the interpreter. Must be
0028     defined in global scope. The first macro parameter is the name of the
0029     module (without quotes). The second parameter is the variable which will
0030     be used as the interface to add functions and classes to the module.
0031
0032     .. code-block:: cpp
0033
0034         PYBIND11_EMBEDDED_MODULE(example, m) {
0035             // ... initialize functions and classes here
0036             m.def("foo", []() {
0037                 return "Hello, World!";
0038             });
0039         }
0040
0041     The third and subsequent macro arguments are optional, and can be used to
0042     mark the module as supporting various Python features.
0043
0044     - ``mod_gil_not_used()``
0045     - ``multiple_interpreters::per_interpreter_gil()``
0046     - ``multiple_interpreters::shared_gil()``
0047     - ``multiple_interpreters::not_supported()``
0048
0049     .. code-block:: cpp
0050
0051         PYBIND11_EMBEDDED_MODULE(example, m, py::mod_gil_not_used()) {
0052             m.def("foo", []() {
0053                 return "Hello, Free-threaded World!";
0054             });
0055         }
0056
0057  \endrst */
0058 PYBIND11_WARNING_PUSH
0059 PYBIND11_WARNING_DISABLE_CLANG("-Wgnu-zero-variadic-macro-arguments")
0060 #define PYBIND11_EMBEDDED_MODULE(name, variable, ...)                                             \
0061     PYBIND11_MODULE_PYINIT(name, {}, ##__VA_ARGS__)                                               \
0062     ::pybind11::detail::embedded_module PYBIND11_CONCAT(pybind11_module_, name)(                  \
0063         PYBIND11_TOSTRING(name), PYBIND11_CONCAT(PyInit_, name));                                 \
0064     PYBIND11_MODULE_EXEC(name, variable)
0065 PYBIND11_WARNING_POP
0066
0067 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
0068 PYBIND11_NAMESPACE_BEGIN(detail)
0069
0070 /// Python 2.7/3.x compatible version of `PyImport_AppendInittab` and error checks.
0071 struct embedded_module {
0072     using init_t = PyObject *(*) ();
0073     embedded_module(const char *name, init_t init) {
0074         if (Py_IsInitialized() != 0) {
0075             pybind11_fail("Can't add new modules after the interpreter has been initialized");
0076         }
0077
0078         auto result = PyImport_AppendInittab(name, init);
0079         if (result == -1) {
0080             pybind11_fail("Insufficient memory to add a new module");
0081         }
0082     }
0083 };
0084
0085 struct wide_char_arg_deleter {
0086     void operator()(wchar_t *ptr) const {
0087         // API docs: https://docs.python.org/3/c-api/sys.html#c.Py_DecodeLocale
0088         PyMem_RawFree(ptr);
0089     }
0090 };
0091
0092 inline wchar_t *widen_chars(const char *safe_arg) {
0093     wchar_t *widened_arg = Py_DecodeLocale(safe_arg, nullptr);
0094     return widened_arg;
0095 }
0096
0097 inline void precheck_interpreter() {
0098     if (Py_IsInitialized() != 0) {
0099         pybind11_fail("The interpreter is already running");
0100     }
0101 }
0102
0103 #if !defined(PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX)
0104 #    define PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX (0x03080000)
0105 #endif
0106
0107 #if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
0108 inline void initialize_interpreter_pre_pyconfig(bool init_signal_handlers,
0109                                                 int argc,
0110                                                 const char *const *argv,
0111                                                 bool add_program_dir_to_path) {
0112     detail::precheck_interpreter();
0113     Py_InitializeEx(init_signal_handlers ? 1 : 0);
0114
0115     auto argv_size = static_cast<size_t>(argc);
0116     // SetArgv* on python 3 takes wchar_t, so we have to convert.
0117     std::unique_ptr<wchar_t *[]> widened_argv(new wchar_t *[argv_size]);
0118     std::vector<std::unique_ptr<wchar_t[], detail::wide_char_arg_deleter>> widened_argv_entries;
0119     widened_argv_entries.reserve(argv_size);
0120     for (size_t ii = 0; ii < argv_size; ++ii) {
0121         widened_argv_entries.emplace_back(detail::widen_chars(argv[ii]));
0122         if (!widened_argv_entries.back()) {
0123             // A null here indicates a character-encoding failure or the python
0124             // interpreter out of memory. Give up.
0125             return;
0126         }
0127         widened_argv[ii] = widened_argv_entries.back().get();
0128     }
0129
0130     auto *pysys_argv = widened_argv.get();
0131
0132     PySys_SetArgvEx(argc, pysys_argv, static_cast<int>(add_program_dir_to_path));
0133 }
0134 #endif
0135
0136 PYBIND11_NAMESPACE_END(detail)
0137
0138 #if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
0139 inline void initialize_interpreter(PyConfig *config,
0140                                    int argc = 0,
0141                                    const char *const *argv = nullptr,
0142                                    bool add_program_dir_to_path = true) {
0143     detail::precheck_interpreter();
0144     PyStatus status = PyConfig_SetBytesArgv(config, argc, const_cast<char *const *>(argv));
0145     if (PyStatus_Exception(status) != 0) {
0146         // A failure here indicates a character-encoding failure or the python
0147         // interpreter out of memory. Give up.
0148         PyConfig_Clear(config);
0149         throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg
0150                                                                : "Failed to prepare CPython");
0151     }
0152     status = Py_InitializeFromConfig(config);
0153     if (PyStatus_Exception(status) != 0) {
0154         PyConfig_Clear(config);
0155         throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg
0156                                                                : "Failed to init CPython");
0157     }
0158     if (add_program_dir_to_path) {
0159         PyRun_SimpleString("import sys, os.path; "
0160                            "sys.path.insert(0, "
0161                            "os.path.abspath(os.path.dirname(sys.argv[0])) "
0162                            "if sys.argv and os.path.exists(sys.argv[0]) else '')");
0163     }
0164     PyConfig_Clear(config);
0165 }
0166 #endif
0167
0168 /** \rst
0169     Initialize the Python interpreter. No other pybind11 or CPython API functions can be
0170     called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The
0171     optional `init_signal_handlers` parameter can be used to skip the registration of
0172     signal handlers (see the `Python documentation`_ for details). Calling this function
0173     again after the interpreter has already been initialized is a fatal error.
0174
0175     If initializing the Python interpreter fails, then the program is terminated.  (This
0176     is controlled by the CPython runtime and is an exception to pybind11's normal behavior
0177     of throwing exceptions on errors.)
0178
0179     The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are
0180     used to populate ``sys.argv`` and ``sys.path``.
0181     See the |PySys_SetArgvEx documentation|_ for details.
0182
0183     .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx
0184     .. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation
0185     .. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx
0186  \endrst */
0187 inline void initialize_interpreter(bool init_signal_handlers = true,
0188                                    int argc = 0,
0189                                    const char *const *argv = nullptr,
0190                                    bool add_program_dir_to_path = true) {
0191 #if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
0192     detail::initialize_interpreter_pre_pyconfig(
0193         init_signal_handlers, argc, argv, add_program_dir_to_path);
0194 #else
0195     PyConfig config;
0196     PyConfig_InitPythonConfig(&config);
0197     // See PR #4473 for background
0198     config.parse_argv = 0;
0199
0200     config.install_signal_handlers = init_signal_handlers ? 1 : 0;
0201     initialize_interpreter(&config, argc, argv, add_program_dir_to_path);
0202 #endif
0203
0204     // There is exactly one interpreter alive currently.
0205     detail::get_num_interpreters_seen() = 1;
0206 }
0207
0208 /** \rst
0209     Shut down the Python interpreter. No pybind11 or CPython API functions can be called
0210     after this. In addition, pybind11 objects must not outlive the interpreter:
0211
0212     .. code-block:: cpp
0213
0214         { // BAD
0215             py::initialize_interpreter();
0216             auto hello = py::str("Hello, World!");
0217             py::finalize_interpreter();
0218         } // <-- BOOM, hello's destructor is called after interpreter shutdown
0219
0220         { // GOOD
0221             py::initialize_interpreter();
0222             { // scoped
0223                 auto hello = py::str("Hello, World!");
0224             } // <-- OK, hello is cleaned up properly
0225             py::finalize_interpreter();
0226         }
0227
0228         { // BETTER
0229             py::scoped_interpreter guard{};
0230             auto hello = py::str("Hello, World!");
0231         }
0232
0233     .. warning::
0234
0235         The interpreter can be restarted by calling `initialize_interpreter` again.
0236         Modules created using pybind11 can be safely re-initialized. However, Python
0237         itself cannot completely unload binary extension modules and there are several
0238         caveats with regard to interpreter restarting. All the details can be found
0239         in the CPython documentation. In short, not all interpreter memory may be
0240         freed, either due to reference cycles or user-created global data.
0241
0242  \endrst */
0243 inline void finalize_interpreter() {
0244     // get rid of any thread-local interpreter cache that currently exists
0245     if (detail::get_num_interpreters_seen() > 1) {
0246         detail::get_internals_pp_manager().unref();
0247         detail::get_local_internals_pp_manager().unref();
0248
0249         // We know there can be no other interpreter alive now, so we can lower the count
0250         detail::get_num_interpreters_seen() = 1;
0251     }
0252
0253     // Re-fetch the internals pointer-to-pointer (but not the internals itself, which might not
0254     // exist). It's possible for the  internals to be created during Py_Finalize() (e.g. if a
0255     // py::capsule calls `get_internals()` during destruction), so we get the pointer-pointer here
0256     // and check it after Py_Finalize().
0257     detail::get_internals_pp_manager().get_pp();
0258     detail::get_local_internals_pp_manager().get_pp();
0259
0260     Py_Finalize();
0261
0262     detail::get_internals_pp_manager().destroy();
0263
0264     // Local internals contains data managed by the current interpreter, so we must clear them to
0265     // avoid undefined behaviors when initializing another interpreter
0266     detail::get_local_internals_pp_manager().destroy();
0267
0268     // We know there is no interpreter alive now, so we can reset the count
0269     detail::get_num_interpreters_seen() = 0;
0270 }
0271
0272 /** \rst
0273     Scope guard version of `initialize_interpreter` and `finalize_interpreter`.
0274     This a move-only guard and only a single instance can exist.
0275
0276     See `initialize_interpreter` for a discussion of its constructor arguments.
0277
0278     .. code-block:: cpp
0279
0280         #include <pybind11/embed.h>
0281
0282         int main() {
0283             py::scoped_interpreter guard{};
0284             py::print(Hello, World!);
0285         } // <-- interpreter shutdown
0286  \endrst */
0287 class scoped_interpreter {
0288 public:
0289     explicit scoped_interpreter(bool init_signal_handlers = true,
0290                                 int argc = 0,
0291                                 const char *const *argv = nullptr,
0292                                 bool add_program_dir_to_path = true) {
0293         initialize_interpreter(init_signal_handlers, argc, argv, add_program_dir_to_path);
0294     }
0295
0296 #if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
0297     explicit scoped_interpreter(PyConfig *config,
0298                                 int argc = 0,
0299                                 const char *const *argv = nullptr,
0300                                 bool add_program_dir_to_path = true) {
0301         initialize_interpreter(config, argc, argv, add_program_dir_to_path);
0302     }
0303 #endif
0304
0305     scoped_interpreter(const scoped_interpreter &) = delete;
0306     scoped_interpreter(scoped_interpreter &&other) noexcept { other.is_valid = false; }
0307     scoped_interpreter &operator=(const scoped_interpreter &) = delete;
0308     scoped_interpreter &operator=(scoped_interpreter &&) = delete;
0309
0310     ~scoped_interpreter() {
0311         if (is_valid) {
0312             finalize_interpreter();
0313         }
0314     }
0315
0316 private:
0317     bool is_valid = true;
0318 };
0319
0320 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)