使用 pybind11 动态加载 libpython

如何解决使用 pybind11 动态加载 libpython

我正在尝试使用 Mac OSX 的 pybind11 构建一些共享库。我遇到了错误：

dyld: Symbol not found: _PyBaSEObject_Type
  Referenced from: /Users/xxxxx/work/test_dynamic_linking/./example
  Expected in: flat namespace
 in /Users/xxxxx/work/test_dynamic_linking/./example
Abort trap: 6

我想要实现的是关闭构建时链接，但在运行时使用 dlopen 动态加载 libpython。注意 cmake 文件中的 -Wl,-undefined,dynamic_lookup 标志。我这样做是因为我想构建一个轮子，并且链接到 libpython 不是一个好主意 AFAIU。

下面是一个可重现的最小示例。我很困惑，如果你直接从 main.cpp 调用 Py_DecodeLocale() 或 Py_InitializeEx() 之类的函数，它工作正常。但是调用 pybind11::initialize_interpreter() 失败并出现上述错误。如果我这样做

nm -gU /opt/anaconda3/envs/py38/lib/libpython3.8.dylib | grep PyBaSEObject

符号 _PyBaSEObject_Type 确实是在 lib 中定义的：

0000000000334528 D _PyBaSEObject_Type

如果我创建一个包装器共享库来包装对 pybind11 函数的调用，并从 main.cpp dlopen 它，它工作正常。这让我更糊涂了。

Cmake 文件：

cmake_minimum_required(VERSION 3.4)
project(example)
set (CMAKE_CXX_STANDARD 11)
set(UNDEFINED_SYMBOLS_IGnorE_FLAG "-Wl,dynamic_lookup")
string(APPEND CMAKE_EXE_LINKER_FLAGS " ${UNDEFINED_SYMBOLS_IGnorE_FLAG}")
string(APPEND CMAKE_SHARED_LINKER_FLAGS " ${UNDEFINED_SYMBOLS_IGnorE_FLAG}")
include_directories(pybind11/include)
include_directories(/opt/anaconda3/envs/py38/include/python3.8)
add_library(pywrapper SHARED ${CMAKE_CURRENT_SOURCE_DIR}/wrapper.cpp)
add_executable(example main.cpp)

pyembed.hpp：

#pragma once

#include "pybind11/embed.h"

namespace py = pybind11;

void initialize_interpreter_func();


struct pybind_wrap_api {
    decltype(&initialize_interpreter_func) initialize_interpreter;
};

wrapper.cpp：

#include "pyembed.hpp"

#include <set>
#include <vector>
#include <iostream>

#include "pybind11/embed.h"
#include "pybind11/stl.h"

namespace py = pybind11;


void initialize_interpreter_func() {
    pybind11::initialize_interpreter();
}

pybind_wrap_api init_pybind_wrap_api() noexcept {
    return {
        &initialize_interpreter_func,};
}

__attribute__((visibility("default"))) pybind_wrap_api pybind_wrapper_api =
    init_pybind_wrap_api();

main.cpp：

#include <pybind11/embed.h> // everything needed for embedding                                                                                                    

#include "pyembed.hpp"

#include <stdlib.h>
#include <dlfcn.h>
#include <iostream>
#include <string>

namespace py = pybind11;
static void* pylib_handle = nullptr;
static void* pybind_wrapper_handle = nullptr;
pybind_wrap_api* wrappers = nullptr;

int main() {
    std::string path_libpython = "/opt/anaconda3/envs/py38/lib/libpython3.8.dylib";
    pylib_handle = dlopen(path_libpython.c_str(),RTLD_Now | RTLD_GLOBAL);
    if(!pylib_handle) {
        std::cout << "load libpython Failed..." << std::endl;
    } else {
        std::cout << "load libpython succeeded..." << std::endl;
    }

    std::string path_wrapper = "./libpywrapper.dylib";
    pybind_wrapper_handle = dlopen(path_wrapper.c_str(),RTLD_Now | RTLD_GLOBAL);
    wrappers = static_cast<pybind_wrap_api*>(dlsym(pybind_wrapper_handle,"pybind_wrapper_api"));

    std::string pythonhome = "/opt/anaconda3/envs/py38";
    setenv("PYTHONHOME",pythonhome.c_str(),1);
    std::string pythonpath = "/opt/anaconda3/envs/py38/lib/python3.8/site-packages";
    setenv("PYTHONPATH",pythonpath.c_str(),true);

    // this line will cause it to fail with the symbol not found error
    py::initialize_interpreter();
    // if comment out the prevIoUs line and do the following line,it works fine. I'm confused why is so. 
    //wrappers->initialize_interpreter();

    return 0; 
}

然后做

cmake . && make && ./example