include/pybind11/embed.h

0001 /*
0002     pybind11/embed.h: Support for embedding the interpreter
0003
0004     Copyright (c) 2017 Wenzel Jakob <wenzel.jakob@epfl.ch>
0005
0006     All rights reserved. Use of this source code is governed by a
0007     BSD-style license that can be found in the LICENSE file.
0008 */
0009
0010 #pragma once
0011
0012 #include "pybind11.h"
0013 #include "eval.h"
0014
0015 #include <memory>
0016 #include <vector>
0017
0018 #if defined(PYPY_VERSION)
0019 #    error Embedding the interpreter is not supported with PyPy
0020 #endif
0021
0022 #define PYBIND11_EMBEDDED_MODULE_IMPL(name)                                                       \
0023     extern "C" PyObject *pybind11_init_impl_##name();                                             \
0024     extern "C" PyObject *pybind11_init_impl_##name() { return pybind11_init_wrapper_##name(); }
0025
0026 /** \rst
0027     Add a new module to the table of builtins for the interpreter. Must be
0028     defined in global scope. The first macro parameter is the name of the
0029     module (without quotes). The second parameter is the variable which will
0030     be used as the interface to add functions and classes to the module.
0031
0032     .. code-block:: cpp
0033
0034         PYBIND11_EMBEDDED_MODULE(example, m) {
0035             // ... initialize functions and classes here
0036             m.def("foo", []() {
0037                 return "Hello, World!";
0038             });
0039         }
0040  \endrst */
0041 #define PYBIND11_EMBEDDED_MODULE(name, variable)                                                  \
0042     static ::pybind11::module_::module_def PYBIND11_CONCAT(pybind11_module_def_, name);           \
0043     static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &);                     \
0044     static PyObject PYBIND11_CONCAT(*pybind11_init_wrapper_, name)() {                            \
0045         auto m = ::pybind11::module_::create_extension_module(                                    \
0046             PYBIND11_TOSTRING(name), nullptr, &PYBIND11_CONCAT(pybind11_module_def_, name));      \
0047         try {                                                                                     \
0048             PYBIND11_CONCAT(pybind11_init_, name)(m);                                             \
0049             return m.ptr();                                                                       \
0050         }                                                                                         \
0051         PYBIND11_CATCH_INIT_EXCEPTIONS                                                            \
0052     }                                                                                             \
0053     PYBIND11_EMBEDDED_MODULE_IMPL(name)                                                           \
0054     ::pybind11::detail::embedded_module PYBIND11_CONCAT(pybind11_module_, name)(                  \
0055         PYBIND11_TOSTRING(name), PYBIND11_CONCAT(pybind11_init_impl_, name));                     \
0056     void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_                                \
0057                                                & variable) // NOLINT(bugprone-macro-parentheses)
0058
0059 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
0060 PYBIND11_NAMESPACE_BEGIN(detail)
0061
0062 /// Python 2.7/3.x compatible version of `PyImport_AppendInittab` and error checks.
0063 struct embedded_module {
0064     using init_t = PyObject *(*) ();
0065     embedded_module(const char *name, init_t init) {
0066         if (Py_IsInitialized() != 0) {
0067             pybind11_fail("Can't add new modules after the interpreter has been initialized");
0068         }
0069
0070         auto result = PyImport_AppendInittab(name, init);
0071         if (result == -1) {
0072             pybind11_fail("Insufficient memory to add a new module");
0073         }
0074     }
0075 };
0076
0077 struct wide_char_arg_deleter {
0078     void operator()(wchar_t *ptr) const {
0079         // API docs: https://docs.python.org/3/c-api/sys.html#c.Py_DecodeLocale
0080         PyMem_RawFree(ptr);
0081     }
0082 };
0083
0084 inline wchar_t *widen_chars(const char *safe_arg) {
0085     wchar_t *widened_arg = Py_DecodeLocale(safe_arg, nullptr);
0086     return widened_arg;
0087 }
0088
0089 inline void precheck_interpreter() {
0090     if (Py_IsInitialized() != 0) {
0091         pybind11_fail("The interpreter is already running");
0092     }
0093 }
0094
0095 #if !defined(PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX)
0096 #    define PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX (0x03080000)
0097 #endif
0098
0099 #if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
0100 inline void initialize_interpreter_pre_pyconfig(bool init_signal_handlers,
0101                                                 int argc,
0102                                                 const char *const *argv,
0103                                                 bool add_program_dir_to_path) {
0104     detail::precheck_interpreter();
0105     Py_InitializeEx(init_signal_handlers ? 1 : 0);
0106 #    if defined(WITH_THREAD) && PY_VERSION_HEX < 0x03070000
0107     PyEval_InitThreads();
0108 #    endif
0109
0110     // Before it was special-cased in python 3.8, passing an empty or null argv
0111     // caused a segfault, so we have to reimplement the special case ourselves.
0112     bool special_case = (argv == nullptr || argc <= 0);
0113
0114     const char *const empty_argv[]{"\0"};
0115     const char *const *safe_argv = special_case ? empty_argv : argv;
0116     if (special_case) {
0117         argc = 1;
0118     }
0119
0120     auto argv_size = static_cast<size_t>(argc);
0121     // SetArgv* on python 3 takes wchar_t, so we have to convert.
0122     std::unique_ptr<wchar_t *[]> widened_argv(new wchar_t *[argv_size]);
0123     std::vector<std::unique_ptr<wchar_t[], detail::wide_char_arg_deleter>> widened_argv_entries;
0124     widened_argv_entries.reserve(argv_size);
0125     for (size_t ii = 0; ii < argv_size; ++ii) {
0126         widened_argv_entries.emplace_back(detail::widen_chars(safe_argv[ii]));
0127         if (!widened_argv_entries.back()) {
0128             // A null here indicates a character-encoding failure or the python
0129             // interpreter out of memory. Give up.
0130             return;
0131         }
0132         widened_argv[ii] = widened_argv_entries.back().get();
0133     }
0134
0135     auto *pysys_argv = widened_argv.get();
0136
0137     PySys_SetArgvEx(argc, pysys_argv, static_cast<int>(add_program_dir_to_path));
0138 }
0139 #endif
0140
0141 PYBIND11_NAMESPACE_END(detail)
0142
0143 #if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
0144 inline void initialize_interpreter(PyConfig *config,
0145                                    int argc = 0,
0146                                    const char *const *argv = nullptr,
0147                                    bool add_program_dir_to_path = true) {
0148     detail::precheck_interpreter();
0149     PyStatus status = PyConfig_SetBytesArgv(config, argc, const_cast<char *const *>(argv));
0150     if (PyStatus_Exception(status) != 0) {
0151         // A failure here indicates a character-encoding failure or the python
0152         // interpreter out of memory. Give up.
0153         PyConfig_Clear(config);
0154         throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg
0155                                                                : "Failed to prepare CPython");
0156     }
0157     status = Py_InitializeFromConfig(config);
0158     if (PyStatus_Exception(status) != 0) {
0159         PyConfig_Clear(config);
0160         throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg
0161                                                                : "Failed to init CPython");
0162     }
0163     if (add_program_dir_to_path) {
0164         PyRun_SimpleString("import sys, os.path; "
0165                            "sys.path.insert(0, "
0166                            "os.path.abspath(os.path.dirname(sys.argv[0])) "
0167                            "if sys.argv and os.path.exists(sys.argv[0]) else '')");
0168     }
0169     PyConfig_Clear(config);
0170 }
0171 #endif
0172
0173 /** \rst
0174     Initialize the Python interpreter. No other pybind11 or CPython API functions can be
0175     called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The
0176     optional `init_signal_handlers` parameter can be used to skip the registration of
0177     signal handlers (see the `Python documentation`_ for details). Calling this function
0178     again after the interpreter has already been initialized is a fatal error.
0179
0180     If initializing the Python interpreter fails, then the program is terminated.  (This
0181     is controlled by the CPython runtime and is an exception to pybind11's normal behavior
0182     of throwing exceptions on errors.)
0183
0184     The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are
0185     used to populate ``sys.argv`` and ``sys.path``.
0186     See the |PySys_SetArgvEx documentation|_ for details.
0187
0188     .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx
0189     .. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation
0190     .. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx
0191  \endrst */
0192 inline void initialize_interpreter(bool init_signal_handlers = true,
0193                                    int argc = 0,
0194                                    const char *const *argv = nullptr,
0195                                    bool add_program_dir_to_path = true) {
0196 #if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
0197     detail::initialize_interpreter_pre_pyconfig(
0198         init_signal_handlers, argc, argv, add_program_dir_to_path);
0199 #else
0200     PyConfig config;
0201     PyConfig_InitIsolatedConfig(&config);
0202     config.isolated = 0;
0203     config.use_environment = 1;
0204     config.install_signal_handlers = init_signal_handlers ? 1 : 0;
0205     initialize_interpreter(&config, argc, argv, add_program_dir_to_path);
0206 #endif
0207 }
0208
0209 /** \rst
0210     Shut down the Python interpreter. No pybind11 or CPython API functions can be called
0211     after this. In addition, pybind11 objects must not outlive the interpreter:
0212
0213     .. code-block:: cpp
0214
0215         { // BAD
0216             py::initialize_interpreter();
0217             auto hello = py::str("Hello, World!");
0218             py::finalize_interpreter();
0219         } // <-- BOOM, hello's destructor is called after interpreter shutdown
0220
0221         { // GOOD
0222             py::initialize_interpreter();
0223             { // scoped
0224                 auto hello = py::str("Hello, World!");
0225             } // <-- OK, hello is cleaned up properly
0226             py::finalize_interpreter();
0227         }
0228
0229         { // BETTER
0230             py::scoped_interpreter guard{};
0231             auto hello = py::str("Hello, World!");
0232         }
0233
0234     .. warning::
0235
0236         The interpreter can be restarted by calling `initialize_interpreter` again.
0237         Modules created using pybind11 can be safely re-initialized. However, Python
0238         itself cannot completely unload binary extension modules and there are several
0239         caveats with regard to interpreter restarting. All the details can be found
0240         in the CPython documentation. In short, not all interpreter memory may be
0241         freed, either due to reference cycles or user-created global data.
0242
0243  \endrst */
0244 inline void finalize_interpreter() {
0245     handle builtins(PyEval_GetBuiltins());
0246     const char *id = PYBIND11_INTERNALS_ID;
0247
0248     // Get the internals pointer (without creating it if it doesn't exist).  It's possible for the
0249     // internals to be created during Py_Finalize() (e.g. if a py::capsule calls `get_internals()`
0250     // during destruction), so we get the pointer-pointer here and check it after Py_Finalize().
0251     detail::internals **internals_ptr_ptr = detail::get_internals_pp();
0252     // It could also be stashed in builtins, so look there too:
0253     if (builtins.contains(id) && isinstance<capsule>(builtins[id])) {
0254         internals_ptr_ptr = capsule(builtins[id]);
0255     }
0256     // Local internals contains data managed by the current interpreter, so we must clear them to
0257     // avoid undefined behaviors when initializing another interpreter
0258     detail::get_local_internals().registered_types_cpp.clear();
0259     detail::get_local_internals().registered_exception_translators.clear();
0260
0261     Py_Finalize();
0262
0263     if (internals_ptr_ptr) {
0264         delete *internals_ptr_ptr;
0265         *internals_ptr_ptr = nullptr;
0266     }
0267 }
0268
0269 /** \rst
0270     Scope guard version of `initialize_interpreter` and `finalize_interpreter`.
0271     This a move-only guard and only a single instance can exist.
0272
0273     See `initialize_interpreter` for a discussion of its constructor arguments.
0274
0275     .. code-block:: cpp
0276
0277         #include <pybind11/embed.h>
0278
0279         int main() {
0280             py::scoped_interpreter guard{};
0281             py::print(Hello, World!);
0282         } // <-- interpreter shutdown
0283  \endrst */
0284 class scoped_interpreter {
0285 public:
0286     explicit scoped_interpreter(bool init_signal_handlers = true,
0287                                 int argc = 0,
0288                                 const char *const *argv = nullptr,
0289                                 bool add_program_dir_to_path = true) {
0290         initialize_interpreter(init_signal_handlers, argc, argv, add_program_dir_to_path);
0291     }
0292
0293 #if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
0294     explicit scoped_interpreter(PyConfig *config,
0295                                 int argc = 0,
0296                                 const char *const *argv = nullptr,
0297                                 bool add_program_dir_to_path = true) {
0298         initialize_interpreter(config, argc, argv, add_program_dir_to_path);
0299     }
0300 #endif
0301
0302     scoped_interpreter(const scoped_interpreter &) = delete;
0303     scoped_interpreter(scoped_interpreter &&other) noexcept { other.is_valid = false; }
0304     scoped_interpreter &operator=(const scoped_interpreter &) = delete;
0305     scoped_interpreter &operator=(scoped_interpreter &&) = delete;
0306
0307     ~scoped_interpreter() {
0308         if (is_valid) {
0309             finalize_interpreter();
0310         }
0311     }
0312
0313 private:
0314     bool is_valid = true;
0315 };
0316
0317 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)