File 0025-58199-wip-python3.9.patch of Package ceph-ceph-19.2.3
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a145688..1206e2d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -552,10 +552,13 @@ endif (WITH_RADOSGW)
option(WITH_CEPHFS "CephFS is enabled" ON)
if(NOT WIN32)
-# Please specify 3.[0-7] if you want to build with a certain version of python3.
+# Please specify 3.x if you want to build with a certain version of python3.
set(WITH_PYTHON3 "3" CACHE STRING "build with specified python3 version")
find_package(Python3 ${WITH_PYTHON3} EXACT REQUIRED
COMPONENTS Interpreter Development)
+if(Python3_VERSION VERSION_LESS 3.9)
+ message(FATAL_ERROR "${Python3_VERSION} is not supported, please use Python 3.9 and up")
+endif()
option(WITH_MGR "ceph-mgr is enabled" ON)
if(WITH_MGR)
diff --git a/src/mgr/PyModule.cc b/src/mgr/PyModule.cc
index 20234ec..2f557f5 100644
--- a/src/mgr/PyModule.cc
+++ b/src/mgr/PyModule.cc
@@ -47,7 +47,6 @@ std::string PyModule::mgr_store_prefix = "mgr/";
using std::string;
-using std::wstring;
// decode a Python exception into a string
std::string handle_pyerror(
@@ -231,72 +230,6 @@ std::pair<int, std::string> PyModuleConfig::set_config(
}
}
-std::string PyModule::get_site_packages()
-{
- std::stringstream site_packages;
-
- // CPython doesn't auto-add site-packages dirs to sys.path for us,
- // but it does provide a module that we can ask for them.
- auto site_module = PyImport_ImportModule("site");
- ceph_assert(site_module);
-
- auto site_packages_fn = PyObject_GetAttrString(site_module, "getsitepackages");
- if (site_packages_fn != nullptr) {
- auto site_packages_list = PyObject_CallObject(site_packages_fn, nullptr);
- ceph_assert(site_packages_list);
-
- auto n = PyList_Size(site_packages_list);
- for (Py_ssize_t i = 0; i < n; ++i) {
- if (i != 0) {
- site_packages << ":";
- }
- site_packages << PyUnicode_AsUTF8(PyList_GetItem(site_packages_list, i));
- }
-
- Py_DECREF(site_packages_list);
- Py_DECREF(site_packages_fn);
- } else {
- // Fall back to generating our own site-packages paths by imitating
- // what the standard site.py does. This is annoying but it lets us
- // run inside virtualenvs :-/
-
- auto site_packages_fn = PyObject_GetAttrString(site_module, "addsitepackages");
- ceph_assert(site_packages_fn);
-
- auto known_paths = PySet_New(nullptr);
- auto pArgs = PyTuple_Pack(1, known_paths);
- PyObject_CallObject(site_packages_fn, pArgs);
- Py_DECREF(pArgs);
- Py_DECREF(known_paths);
- Py_DECREF(site_packages_fn);
-
- auto sys_module = PyImport_ImportModule("sys");
- ceph_assert(sys_module);
- auto sys_path = PyObject_GetAttrString(sys_module, "path");
- ceph_assert(sys_path);
-
- dout(1) << "sys.path:" << dendl;
- auto n = PyList_Size(sys_path);
- bool first = true;
- for (Py_ssize_t i = 0; i < n; ++i) {
- dout(1) << " " << PyUnicode_AsUTF8(PyList_GetItem(sys_path, i)) << dendl;
- if (first) {
- first = false;
- } else {
- site_packages << ":";
- }
- site_packages << PyUnicode_AsUTF8(PyList_GetItem(sys_path, i));
- }
-
- Py_DECREF(sys_path);
- Py_DECREF(sys_module);
- }
-
- Py_DECREF(site_module);
-
- return site_packages.str();
-}
-
PyObject* PyModule::init_ceph_logger()
{
auto py_logger = PyModule_Create(&ceph_logger_module);
@@ -357,17 +290,6 @@ int PyModule::load(PyThreadState *pMainThreadState)
return -EINVAL;
} else {
pMyThreadState.set(thread_state);
- // Some python modules do not cope with an unpopulated argv, so lets
- // fake one. This step also picks up site-packages into sys.path.
- const wchar_t *argv[] = {L"ceph-mgr"};
- PySys_SetArgv(1, (wchar_t**)argv);
- // Configure sys.path to include mgr_module_path
- string paths = (g_conf().get_val<std::string>("mgr_module_path") + ':' +
- get_site_packages() + ':');
- wstring sys_path(wstring(begin(paths), end(paths)) + Py_GetPath());
- PySys_SetPath(const_cast<wchar_t*>(sys_path.c_str()));
- dout(10) << "Computed sys.path '"
- << string(begin(sys_path), end(sys_path)) << "'" << dendl;
}
}
// Environment is all good, import the external module
diff --git a/src/mgr/PyModule.h b/src/mgr/PyModule.h
index 8d88ff9..177447c 100644
--- a/src/mgr/PyModule.h
+++ b/src/mgr/PyModule.h
@@ -51,7 +51,6 @@ class PyModule
mutable ceph::mutex lock = ceph::make_mutex("PyModule::lock");
private:
const std::string module_name;
- std::string get_site_packages();
int load_subclass_of(const char* class_name, PyObject** py_class);
// Did the MgrMap identify this module as one that should run?
diff --git a/src/mgr/PyModuleRegistry.cc b/src/mgr/PyModuleRegistry.cc
index 407f3f8..4255528 100644
--- a/src/mgr/PyModuleRegistry.cc
+++ b/src/mgr/PyModuleRegistry.cc
@@ -14,6 +14,7 @@
#include "PyModuleRegistry.h"
#include <filesystem>
+#include <boost/scope_exit.hpp>
#include "include/stringify.h"
#include "common/errno.h"
@@ -46,21 +47,51 @@ void PyModuleRegistry::init()
// Set up global python interpreter
#define WCHAR(s) L ## #s
- Py_SetProgramName(const_cast<wchar_t*>(WCHAR(MGR_PYTHON_EXECUTABLE)));
-#undef WCHAR
+ PyConfig py_config;
+ // do not enable isolated mode, otherwise we would not be able to have access
+ // to the site packages. since we cannot import any module before initializing
+ // the interpreter, we would not be able to use "site" module for retrieving
+ // the path to site packager. we import "site" module for retrieving
+ // sitepackages in Python < 3.8 though, this does not apply to the
+ // initialization with PyConfig.
+ PyConfig_InitPythonConfig(&py_config);
+ BOOST_SCOPE_EXIT_ALL(&py_config) {
+ PyConfig_Clear(&py_config);
+ };
+#if PY_VERSION_HEX >= 0x030b0000
+ py_config.safe_path = 0;
+#endif
+ py_config.parse_argv = 0;
+ py_config.configure_c_stdio = 0;
+ py_config.install_signal_handlers = 0;
+ py_config.pathconfig_warnings = 0;
+
+ PyStatus status;
+ status = PyConfig_SetString(&py_config, &py_config.program_name, WCHAR(MGR_PYTHON_EXECUTABLE));
+ ceph_assertf(!PyStatus_Exception(status), "PyConfig_SetString: %s:%s", status.func, status.err_msg);
+ // Some python modules do not cope with an unpopulated argv, so lets
+ // fake one. This step also picks up site-packages into sys.path.
+ const wchar_t* argv[] = {L"ceph-mgr"};
+ status = PyConfig_SetArgv(&py_config, 1, (wchar_t *const *)argv);
+ ceph_assertf(!PyStatus_Exception(status), "PyConfig_SetArgv: %s:%s", status.func, status.err_msg);
// Add more modules
if (g_conf().get_val<bool>("daemonize")) {
PyImport_AppendInittab("ceph_logger", PyModule::init_ceph_logger);
}
PyImport_AppendInittab("ceph_module", PyModule::init_ceph_module);
- Py_InitializeEx(0);
-#if PY_VERSION_HEX < 0x03090000
- // Let CPython know that we will be calling it back from other
- // threads in future.
- if (! PyEval_ThreadsInitialized()) {
- PyEval_InitThreads();
+ // Configure sys.path to include mgr_module_path
+ auto pythonpath_env = g_conf().get_val<std::string>("mgr_module_path");
+ if (const char* pythonpath = getenv("PYTHONPATH")) {
+ pythonpath_env += ":";
+ pythonpath_env += pythonpath;
}
-#endif
+ status = PyConfig_SetBytesString(&py_config, &py_config.pythonpath_env, pythonpath_env.data());
+ ceph_assertf(!PyStatus_Exception(status), "PyConfig_SetBytesString: %s:%s", status.func, status.err_msg);
+ dout(10) << "set PYTHONPATH to " << std::quoted(pythonpath_env) << dendl;
+ status = Py_InitializeFromConfig(&py_config);
+ ceph_assertf(!PyStatus_Exception(status), "Py_InitializeFromConfig: %s:%s", status.func, status.err_msg);
+#undef WCHAR
+
// Drop the GIL and remember the main thread state (current
// thread state becomes NULL)
pMainThreadState = PyEval_SaveThread();
@@ -228,6 +259,72 @@ void PyModuleRegistry::active_start(
}
}
+std::string PyModuleRegistry::get_site_packages()
+{
+ std::stringstream site_packages;
+
+ // CPython doesn't auto-add site-packages dirs to sys.path for us,
+ // but it does provide a module that we can ask for them.
+ auto site_module = PyImport_ImportModule("site");
+ ceph_assert(site_module);
+
+ auto site_packages_fn = PyObject_GetAttrString(site_module, "getsitepackages");
+ if (site_packages_fn != nullptr) {
+ auto site_packages_list = PyObject_CallObject(site_packages_fn, nullptr);
+ ceph_assert(site_packages_list);
+
+ auto n = PyList_Size(site_packages_list);
+ for (Py_ssize_t i = 0; i < n; ++i) {
+ if (i != 0) {
+ site_packages << ":";
+ }
+ site_packages << PyUnicode_AsUTF8(PyList_GetItem(site_packages_list, i));
+ }
+
+ Py_DECREF(site_packages_list);
+ Py_DECREF(site_packages_fn);
+ } else {
+ // Fall back to generating our own site-packages paths by imitating
+ // what the standard site.py does. This is annoying but it lets us
+ // run inside virtualenvs :-/
+
+ auto site_packages_fn = PyObject_GetAttrString(site_module, "addsitepackages");
+ ceph_assert(site_packages_fn);
+
+ auto known_paths = PySet_New(nullptr);
+ auto pArgs = PyTuple_Pack(1, known_paths);
+ PyObject_CallObject(site_packages_fn, pArgs);
+ Py_DECREF(pArgs);
+ Py_DECREF(known_paths);
+ Py_DECREF(site_packages_fn);
+
+ auto sys_module = PyImport_ImportModule("sys");
+ ceph_assert(sys_module);
+ auto sys_path = PyObject_GetAttrString(sys_module, "path");
+ ceph_assert(sys_path);
+
+ dout(1) << "sys.path:" << dendl;
+ auto n = PyList_Size(sys_path);
+ bool first = true;
+ for (Py_ssize_t i = 0; i < n; ++i) {
+ dout(1) << " " << PyUnicode_AsUTF8(PyList_GetItem(sys_path, i)) << dendl;
+ if (first) {
+ first = false;
+ } else {
+ site_packages << ":";
+ }
+ site_packages << PyUnicode_AsUTF8(PyList_GetItem(sys_path, i));
+ }
+
+ Py_DECREF(sys_path);
+ Py_DECREF(sys_module);
+ }
+
+ Py_DECREF(site_module);
+
+ return site_packages.str();
+}
+
std::vector<std::string> PyModuleRegistry::probe_modules(const std::string &path) const
{
const auto opt = g_conf().get_val<std::string>("mgr_disabled_modules");
diff --git a/src/mgr/PyModuleRegistry.h b/src/mgr/PyModuleRegistry.h
index b5549e6..a89ae6c 100644
--- a/src/mgr/PyModuleRegistry.h
+++ b/src/mgr/PyModuleRegistry.h
@@ -56,6 +56,7 @@ private:
// before ClusterState exists.
MgrMap mgr_map;
+ static std::string get_site_packages();
/**
* Discover python modules from local disk
*/
