include/pybind11/embed.h

0001 /*
0002     pybind11/embed.h: Support for embedding the interpreter
0003
0004     Copyright (c) 2017 Wenzel Jakob <wenzel.jakob@epfl.ch>
0005
0006     All rights reserved. Use of this source code is governed by a
0007     BSD-style license that can be found in the LICENSE file.
0008 */
0009
0010 #pragma once
0011
0012 #include "pybind11.h"
0013 #include "eval.h"
0014
0015 #include <memory>
0016 #include <vector>
0017
0018 #if defined(PYPY_VERSION)
0019 #    error Embedding the interpreter is not supported with PyPy
0020 #endif
0021
0022 #define PYBIND11_EMBEDDED_MODULE_IMPL(name)                                                       \
0023     extern "C" PyObject *pybind11_init_impl_##name();                                             \
0024     extern "C" PyObject *pybind11_init_impl_##name() { return pybind11_init_wrapper_##name(); }
0025
0026 /** \rst
0027     Add a new module to the table of builtins for the interpreter. Must be
0028     defined in global scope. The first macro parameter is the name of the
0029     module (without quotes). The second parameter is the variable which will
0030     be used as the interface to add functions and classes to the module.
0031
0032     .. code-block:: cpp
0033
0034         PYBIND11_EMBEDDED_MODULE(example, m) {
0035             // ... initialize functions and classes here
0036             m.def("foo", []() {
0037                 return "Hello, World!";
0038             });
0039         }
0040  \endrst */
0041 #define PYBIND11_EMBEDDED_MODULE(name, variable)                                                  \
0042     static ::pybind11::module_::module_def PYBIND11_CONCAT(pybind11_module_def_, name);           \
0043     static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &);                     \
0044     static PyObject PYBIND11_CONCAT(*pybind11_init_wrapper_, name)() {                            \
0045         auto m = ::pybind11::module_::create_extension_module(                                    \
0046             PYBIND11_TOSTRING(name), nullptr, &PYBIND11_CONCAT(pybind11_module_def_, name));      \
0047         try {                                                                                     \
0048             PYBIND11_CONCAT(pybind11_init_, name)(m);                                             \
0049             return m.ptr();                                                                       \
0050         }                                                                                         \
0051         PYBIND11_CATCH_INIT_EXCEPTIONS                                                            \
0052     }                                                                                             \
0053     PYBIND11_EMBEDDED_MODULE_IMPL(name)                                                           \
0054     ::pybind11::detail::embedded_module PYBIND11_CONCAT(pybind11_module_, name)(                  \
0055         PYBIND11_TOSTRING(name), PYBIND11_CONCAT(pybind11_init_impl_, name));                     \
0056     void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_                                \
0057                                                & variable) // NOLINT(bugprone-macro-parentheses)
0058
0059 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
0060 PYBIND11_NAMESPACE_BEGIN(detail)
0061
0062 /// Python 2.7/3.x compatible version of `PyImport_AppendInittab` and error checks.
0063 struct embedded_module {
0064     using init_t = PyObject *(*) ();
0065     embedded_module(const char *name, init_t init) {
0066         if (Py_IsInitialized() != 0) {
0067             pybind11_fail("Can't add new modules after the interpreter has been initialized");
0068         }
0069
0070         auto result = PyImport_AppendInittab(name, init);
0071         if (result == -1) {
0072             pybind11_fail("Insufficient memory to add a new module");
0073         }
0074     }
0075 };
0076
0077 struct wide_char_arg_deleter {
0078     void operator()(wchar_t *ptr) const {
0079         // API docs: https://docs.python.org/3/c-api/sys.html#c.Py_DecodeLocale
0080         PyMem_RawFree(ptr);
0081     }
0082 };
0083
0084 inline wchar_t *widen_chars(const char *safe_arg) {
0085     wchar_t *widened_arg = Py_DecodeLocale(safe_arg, nullptr);
0086     return widened_arg;
0087 }
0088
0089 inline void precheck_interpreter() {
0090     if (Py_IsInitialized() != 0) {
0091         pybind11_fail("The interpreter is already running");
0092     }
0093 }
0094
0095 #if !defined(PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX)
0096 #    define PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX (0x03080000)
0097 #endif
0098
0099 #if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
0100 inline void initialize_interpreter_pre_pyconfig(bool init_signal_handlers,
0101                                                 int argc,
0102                                                 const char *const *argv,
0103                                                 bool add_program_dir_to_path) {
0104     detail::precheck_interpreter();
0105     Py_InitializeEx(init_signal_handlers ? 1 : 0);
0106
0107     // Before it was special-cased in python 3.8, passing an empty or null argv
0108     // caused a segfault, so we have to reimplement the special case ourselves.
0109     bool special_case = (argv == nullptr || argc <= 0);
0110
0111     const char *const empty_argv[]{"\0"};
0112     const char *const *safe_argv = special_case ? empty_argv : argv;
0113     if (special_case) {
0114         argc = 1;
0115     }
0116
0117     auto argv_size = static_cast<size_t>(argc);
0118     // SetArgv* on python 3 takes wchar_t, so we have to convert.
0119     std::unique_ptr<wchar_t *[]> widened_argv(new wchar_t *[argv_size]);
0120     std::vector<std::unique_ptr<wchar_t[], detail::wide_char_arg_deleter>> widened_argv_entries;
0121     widened_argv_entries.reserve(argv_size);
0122     for (size_t ii = 0; ii < argv_size; ++ii) {
0123         widened_argv_entries.emplace_back(detail::widen_chars(safe_argv[ii]));
0124         if (!widened_argv_entries.back()) {
0125             // A null here indicates a character-encoding failure or the python
0126             // interpreter out of memory. Give up.
0127             return;
0128         }
0129         widened_argv[ii] = widened_argv_entries.back().get();
0130     }
0131
0132     auto *pysys_argv = widened_argv.get();
0133
0134     PySys_SetArgvEx(argc, pysys_argv, static_cast<int>(add_program_dir_to_path));
0135 }
0136 #endif
0137
0138 PYBIND11_NAMESPACE_END(detail)
0139
0140 #if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
0141 inline void initialize_interpreter(PyConfig *config,
0142                                    int argc = 0,
0143                                    const char *const *argv = nullptr,
0144                                    bool add_program_dir_to_path = true) {
0145     detail::precheck_interpreter();
0146     PyStatus status = PyConfig_SetBytesArgv(config, argc, const_cast<char *const *>(argv));
0147     if (PyStatus_Exception(status) != 0) {
0148         // A failure here indicates a character-encoding failure or the python
0149         // interpreter out of memory. Give up.
0150         PyConfig_Clear(config);
0151         throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg
0152                                                                : "Failed to prepare CPython");
0153     }
0154     status = Py_InitializeFromConfig(config);
0155     if (PyStatus_Exception(status) != 0) {
0156         PyConfig_Clear(config);
0157         throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg
0158                                                                : "Failed to init CPython");
0159     }
0160     if (add_program_dir_to_path) {
0161         PyRun_SimpleString("import sys, os.path; "
0162                            "sys.path.insert(0, "
0163                            "os.path.abspath(os.path.dirname(sys.argv[0])) "
0164                            "if sys.argv and os.path.exists(sys.argv[0]) else '')");
0165     }
0166     PyConfig_Clear(config);
0167 }
0168 #endif
0169
0170 /** \rst
0171     Initialize the Python interpreter. No other pybind11 or CPython API functions can be
0172     called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The
0173     optional `init_signal_handlers` parameter can be used to skip the registration of
0174     signal handlers (see the `Python documentation`_ for details). Calling this function
0175     again after the interpreter has already been initialized is a fatal error.
0176
0177     If initializing the Python interpreter fails, then the program is terminated.  (This
0178     is controlled by the CPython runtime and is an exception to pybind11's normal behavior
0179     of throwing exceptions on errors.)
0180
0181     The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are
0182     used to populate ``sys.argv`` and ``sys.path``.
0183     See the |PySys_SetArgvEx documentation|_ for details.
0184
0185     .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx
0186     .. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation
0187     .. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx
0188  \endrst */
0189 inline void initialize_interpreter(bool init_signal_handlers = true,
0190                                    int argc = 0,
0191                                    const char *const *argv = nullptr,
0192                                    bool add_program_dir_to_path = true) {
0193 #if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
0194     detail::initialize_interpreter_pre_pyconfig(
0195         init_signal_handlers, argc, argv, add_program_dir_to_path);
0196 #else
0197     PyConfig config;
0198     PyConfig_InitPythonConfig(&config);
0199     // See PR #4473 for background
0200     config.parse_argv = 0;
0201
0202     config.install_signal_handlers = init_signal_handlers ? 1 : 0;
0203     initialize_interpreter(&config, argc, argv, add_program_dir_to_path);
0204 #endif
0205 }
0206
0207 /** \rst
0208     Shut down the Python interpreter. No pybind11 or CPython API functions can be called
0209     after this. In addition, pybind11 objects must not outlive the interpreter:
0210
0211     .. code-block:: cpp
0212
0213         { // BAD
0214             py::initialize_interpreter();
0215             auto hello = py::str("Hello, World!");
0216             py::finalize_interpreter();
0217         } // <-- BOOM, hello's destructor is called after interpreter shutdown
0218
0219         { // GOOD
0220             py::initialize_interpreter();
0221             { // scoped
0222                 auto hello = py::str("Hello, World!");
0223             } // <-- OK, hello is cleaned up properly
0224             py::finalize_interpreter();
0225         }
0226
0227         { // BETTER
0228             py::scoped_interpreter guard{};
0229             auto hello = py::str("Hello, World!");
0230         }
0231
0232     .. warning::
0233
0234         The interpreter can be restarted by calling `initialize_interpreter` again.
0235         Modules created using pybind11 can be safely re-initialized. However, Python
0236         itself cannot completely unload binary extension modules and there are several
0237         caveats with regard to interpreter restarting. All the details can be found
0238         in the CPython documentation. In short, not all interpreter memory may be
0239         freed, either due to reference cycles or user-created global data.
0240
0241  \endrst */
0242 inline void finalize_interpreter() {
0243     // Get the internals pointer (without creating it if it doesn't exist).  It's possible for the
0244     // internals to be created during Py_Finalize() (e.g. if a py::capsule calls `get_internals()`
0245     // during destruction), so we get the pointer-pointer here and check it after Py_Finalize().
0246     detail::internals **internals_ptr_ptr = detail::get_internals_pp();
0247     // It could also be stashed in state_dict, so look there too:
0248     if (object internals_obj
0249         = get_internals_obj_from_state_dict(detail::get_python_state_dict())) {
0250         internals_ptr_ptr = detail::get_internals_pp_from_capsule(internals_obj);
0251     }
0252     // Local internals contains data managed by the current interpreter, so we must clear them to
0253     // avoid undefined behaviors when initializing another interpreter
0254     detail::get_local_internals().registered_types_cpp.clear();
0255     detail::get_local_internals().registered_exception_translators.clear();
0256
0257     Py_Finalize();
0258
0259     if (internals_ptr_ptr) {
0260         delete *internals_ptr_ptr;
0261         *internals_ptr_ptr = nullptr;
0262     }
0263 }
0264
0265 /** \rst
0266     Scope guard version of `initialize_interpreter` and `finalize_interpreter`.
0267     This a move-only guard and only a single instance can exist.
0268
0269     See `initialize_interpreter` for a discussion of its constructor arguments.
0270
0271     .. code-block:: cpp
0272
0273         #include <pybind11/embed.h>
0274
0275         int main() {
0276             py::scoped_interpreter guard{};
0277             py::print(Hello, World!);
0278         } // <-- interpreter shutdown
0279  \endrst */
0280 class scoped_interpreter {
0281 public:
0282     explicit scoped_interpreter(bool init_signal_handlers = true,
0283                                 int argc = 0,
0284                                 const char *const *argv = nullptr,
0285                                 bool add_program_dir_to_path = true) {
0286         initialize_interpreter(init_signal_handlers, argc, argv, add_program_dir_to_path);
0287     }
0288
0289 #if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
0290     explicit scoped_interpreter(PyConfig *config,
0291                                 int argc = 0,
0292                                 const char *const *argv = nullptr,
0293                                 bool add_program_dir_to_path = true) {
0294         initialize_interpreter(config, argc, argv, add_program_dir_to_path);
0295     }
0296 #endif
0297
0298     scoped_interpreter(const scoped_interpreter &) = delete;
0299     scoped_interpreter(scoped_interpreter &&other) noexcept { other.is_valid = false; }
0300     scoped_interpreter &operator=(const scoped_interpreter &) = delete;
0301     scoped_interpreter &operator=(scoped_interpreter &&) = delete;
0302
0303     ~scoped_interpreter() {
0304         if (is_valid) {
0305             finalize_interpreter();
0306         }
0307     }
0308
0309 private:
0310     bool is_valid = true;
0311 };
0312
0313 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)