I try to pass vector/array by reference from python through pybind11 to a C++ library. The C++ library may fill in data. After the call to C++, I hope the python side will get the data.
Here is the simplified C++ code:
#include <pybind11/pybind11.h>
#include <pybind11/numpy.h>
#include <pybind11/stl.h>
class Setup
{
public:
Setup(int version) : _version(version) {}
int _version;
};
class Calculator
{
public:
Calculator() {}
static void calc(const Setup& setup, std::vector<double>& results) { ... }
}
namespace py = pybind11;
PYBIND11_MODULE(one_calculator, m) {
// optional module docstring
m.doc() = "pybind11 one_calculator plugin";
py::class_<Setup>(m, "Setup")
.def(py::init<int>());
py::class_<Calculator>(m, "Calculator")
.def(py::init<>())
.def("calc", &Calculator::calc);
}
On the python side, I intend to:
import os
import sys
import numpy as np
import pandas as pd
sys.path.append(os.path.realpath('...'))
from one_calculator import Setup, Calculator
a_setup = Setup(1)
a_calculator = Calculator()
results = []
a_calculator.calc(a_setup, results)
results
Apparently the results are not passed back. Is there a neat way to do it?
Figured out a way:
#include <pybind11/pybind11.h>
#include <pybind11/numpy.h>
#include <pybind11/stl.h>
#include "Calculator.h" // where run_calculator is
namespace py = pybind11;
// wrap c++ function with Numpy array IO
int wrapper(const std::string& input_file, py::array_t<double>& in_results) {
if (in_results.ndim() != 2)
throw std::runtime_error("Results should be a 2-D Numpy array");
auto buf = in_results.request();
double* ptr = (double*)buf.ptr;
size_t N = in_results.shape()[0];
size_t M = in_results.shape()[1];
std::vector<std::vector<double> > results;
run_calculator(input_file, results);
size_t pos = 0;
for (size_t i = 0; i < results.size(); i++) {
const std::vector<double>& line_data = results[i];
for (size_t j = 0; j < line_data.size(); j++) {
ptr[pos] = line_data[j];
pos++;
}
}
}
PYBIND11_MODULE(calculator, m) {
// optional module docstring
m.doc() = "pybind11 calculator plugin";
m.def("run_calculator", &wrapper, "Run the calculator");
}
Python side
results= np.zeros((N, M))
run_calculator(input_file, results)
This way I also do not expose classes Setup and Calculator to the python side.
Related
Run into a frustrating issue trying to import a Python library that itself calls some code I wrote in C++ and compiled into a .so
The C++ code is as follows:
#include <iostream>
#include <string>
#include <vector>
#include <algorithm>
#include <numeric>
#include <cmath>
std::vector<std::string> bigram(std::string initial_str) {
int len = initial_str.size();
std::vector<std::string> tokens;
for (int i = 0; i < len-1; i += 1){
tokens.push_back(initial_str.substr(i, 2));
}
return tokens;
}
std::vector<std::string> vsunion(std::vector<std::string> s1, std::vector<std::string> s2) {
std::vector<std::string> union_str(s1);
union_str.insert(union_str.end(), s2.begin(), s2.end());
std::sort(union_str.begin(), union_str.end());
union_str.erase(std::unique(union_str.begin(), union_str.end()), union_str.end());
return union_str;
}
std::vector<int> ufreq(std::vector<std::string> u, std::vector<std::string> s) {
int len = u.size();
std::vector<int> vfreq;
for (int i = 0; i < len; i += 1){
int freq = std::count(s.begin(), s.end(), u[i]);
vfreq.push_back(freq);
}
return vfreq;
}
float similarity(std::vector<int> f1, std::vector<int> f2) {
float num = std::inner_product(f1.begin(), f1.end(), f2.begin(), 0.0);
float den1 = std::inner_product(f1.begin(), f1.end(), f1.begin(), 0.0);
float den2 = std::inner_product(f2.begin(), f2.end(), f2.begin(), 0.0);
float similarity = num / std::sqrt(den1 * den2);
return similarity;
}
float similarity(std::string string1, std::string string2) {
std::vector<std::string> new_str = bigram(string1);
std::vector<std::string> new_str2 = bigram(string2);
std::vector<std::string> union_str = vsunion(new_str, new_str2);
std::vector<int> freq1 = ufreq(union_str, new_str);
std::vector<int> freq2 = ufreq(union_str, new_str2);
float score = similarity(freq1, freq2);
return score;
}
extern "C" {
float gram(std::string str1, std::string str2)
{
return similarity(str1, str2);
}
}
Which I compiled using:
g++ gram.cpp -shared -o gram.so
and finally I'm trying to import the below script that's throwing the error in the title "_ZSt28__throw_bad_array_new_lengthv could not be located in the dynamic link library":
import ctypes
import sys
import os
dir_path = os.path.dirname(os.path.realpath(__file__))
handle = ctypes.CDLL(dir_path + "/gram.so")
handle.My_Function.argtypes = [ctypes.c_wchar_p]
def gram(string1, string2):
return handle.gram(string1, string2)
Any idea where I might have gone wrong? I can compile a test case instead of the "extern" bit:
int main() {
float score = similarity("John Smith", "Johnny John Smith");
std::cout << score << " ";
std::cin.get();
}
and run as an exe, seemingly without issue; something seems to be going wrong at the
handle = ctypes.CDLL(dir_path + "/gram.so")
stage in the gram library.
finally I'm trying to import the below script that's throwing the error in the title "_ZSt28__throw_bad_array_new_lengthv could not be located in the dynamic link library":
This error means: the version of libstdc++.so available at runtime doesn't have std::throw_bad_array_new_length symbol, which your gram.so is using.
While it's possible to work around this with g++ gram.cpp -shared -o gram.so -fPIC -static-libstdc++ -static-libgcc, you really shouldn't -- it will explode in your face sooner or later.
Instead, you should use Python which has been linked against libstdc++.so.6 appropriate for your system.
In python language, we can use A[A!=0]=-10 to trans all the non-zero value in A to -10. How can I implement this function in C++ language or is here any similar function in 3rd party?
#include <opencv2/core.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include <iostream>
using namespace std;
using namespace cv;
int main()
{
Mat A(3, 3, CV_16SC1);
for (int i = 0; i < 3; i++)
{
for (int j = 0; j < 3; j++)
{
A.at<short>(i, j) = i + j;
}
}
for (auto& value : A) if (value != 2) value = -10;
}
There is std::ranges::replace in C++20:
std::vector<int> values = {1,2,3,1,2,5,3,165};
std::ranges::replace(values, 3, 999);
And, similarly, std::ranges::replace_if in C++20:
std::vector<int> values = {1,2,0,1,0,5,3,165};
std::ranges::replace_if(values, [](int a) { return a != 0;}, -10);
You can see it in
compiler explorer
It is also possible to use std::not_equal_to with std::bind_front instead of the lambda, if you prefer it that way, like this
Edit: here is the bind_front code (from the above link):
std::ranges::replace_if(values, std::bind_front(std::not_equal_to{}, 0), -20);
Range-based for loops will not work since the cv::Mat::begin() is a member function template. You'll have to use begin<mat_type>() and end<mat_type>().
Example:
#include <opencv2/core.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include <algorithm>
#include <iostream>
#include <iterator>
int main() {
cv::Mat A(3, 3, CV_16SC1);
using mtype = short; // convenience typedef
for(int i = 0; i < 3; i++) {
for(int j = 0; j < 3; j++) {
A.at<mtype>(i, j) = i + j; // using mtype
}
}
// using mtype:
for(auto it = A.begin<mtype>(); it != A.end<mtype>(); ++it) {
if(*it != 2) *it = -10;
}
// print result:
std::copy(A.begin<mtype>(), A.end<mtype>(),
std::ostream_iterator<mtype>(std::cout, "\n"));
}
Or using the std::replace_if algorithm to replace all non 2's with -10:
std::replace_if(A.begin<mtype>(), A.end<mtype>(), // using mtype
[](auto& value) { return value != 2; }, -10);
Output:
-10
-10
2
-10
2
-10
2
-10
-10
In STL container there is a function that has similar behavior. For example:
#include <vector>
#include <algorithm>
int main() {
using namespace std;
vector<int> example_vector{0,1,4,1,5};
for_each(example_vector.begin(), example_vector.end(), [](auto& a) { if (a != 0)a = -10; });
}
Although the syntax seems not simple like python, it is optimized, and may be parallelized by execution policy.
I'm trying to link Fortran|C/C++|Python using VS2010 on Windows 64-bit. I have a main code that is written in Fortran. From that code I call a C++ function and after that I call a Python function to do some staff further. Here is a simplified version of my code:
!Fmain.f90
PROGRAM fmain
IMPLICIT NONE
INTERFACE
SUBROUTINE Call_NN(input_array, output_array) BIND(C, name='Call_NN')
USE, INTRINSIC :: iso_c_binding
IMPLICIT NONE
REAL(C_DOUBLE), INTENT(IN), DIMENSION(*) :: input_array
REAL(C_DOUBLE), INTENT(INOUT), DIMENSION(*) :: output_array
END SUBROUTINE
END INTERFACE
REAL*8, DIMENSION(0:2) :: input_array, output_array
REAL :: b
INTEGER :: i
do i = 1, 3
input_array = 0.01d0
output_array = 0.d0
call Call_NN(input_array, output_array)
enddo
END
The C++ static library is as following:
//Csub.cpp
#include <Python.h>
#include <cassert>
#include <stdio.h>
#include <conio.h>
#include <iostream>
#include <fstream>
#include <array>
#include <string.h>
#include <errno.h>
#include <limits.h>
#include <assert.h>
#include <stdlib.h>
using namespace std;
extern "C" void Call_NN(array<double,3> input_array, array<double,3> output_array)
{
// Initialize the Python interpreter.
Py_Initialize();
// Variables Declaration
float result = 0;
float result1 = 0;
float result2 = 0;
float result3 = 0;
float value1 = 0;
float value2 = 0;
float value3 = 0;
// Create some Python objects that will later be assigned values.
PyObject *pName, *pModule, *pDict, *pFunc, *pArgs, *pValue1, *pValue2, *pValue3;
PyObject *pT1, *pT2, *pT3;
// Convert the file name to a Python string.
const char* filename = "module";
pName = PyString_FromString(filename);
if (pName == nullptr)
{
PyErr_Print();
std::exit(1);
}
// Import the file as a Python module.
pModule = PyImport_Import(pName);
if (pModule == nullptr)
{
PyErr_Print();
std::exit(1);
}
// Create a dictionary for the contents of the module.
pDict = PyModule_GetDict(pModule);
if (pDict == nullptr)
{
PyErr_Print();
std::exit(1);
}
// Get the add method from the dictionary.
pFunc = PyDict_GetItemString(pDict, "NN");
if (pFunc == nullptr)
{
PyErr_Print();
std::exit(1);
}
// Create a Python tuple to hold the arguments to the method.
pArgs = PyTuple_New(3);
if (pArgs == nullptr)
{
PyErr_Print();
std::exit(1);
}
// Convert 3 to a Python integer.
value1 = input_array[0];
value2 = input_array[1];
value3 = input_array[2];
pValue1 = PyFloat_FromDouble(value1);
pValue2 = PyFloat_FromDouble(value2);
pValue3 = PyFloat_FromDouble(value3);
// Set the Python int as the first and second arguments to the method.
PyTuple_SetItem(pArgs, 0, pValue1);
PyTuple_SetItem(pArgs, 1, pValue2);
PyTuple_SetItem(pArgs, 2, pValue3);
// Call the function with the arguments.
PyObject* pResult = PyObject_CallObject(pFunc, pArgs);
// Print a message if calling the method failed.
if (pResult == NULL)
printf("Calling the add method failed.\n");
// Convert the result to a long from a Python object.
//result = PyFloat_AsDouble(pResult);
pT1 = PyTuple_GetItem(pResult, 0);
pT2 = PyTuple_GetItem(pResult, 1);
pT3 = PyTuple_GetItem(pResult, 2);
// Convert output to float
result1 = PyFloat_AsDouble(pT1);
result2 = PyFloat_AsDouble(pT2);
result3 = PyFloat_AsDouble(pT3);
output_array[0] = result1;
output_array[1] = result2;
output_array[2] = result3;
// Destroy the Python interpreter.
Py_Finalize();
}
And the Python module is defined as:
# module.py
import sys
import numpy as np
import pandas as pd
import pickle
from sklearn.externals import joblib
def NN(a,b,c,):
X_array = np.array([a, b, c])
X = pd.DataFrame(X_array).transpose()
clf = joblib.load('SavedNeuralNetwork.pkl')
y = clf.predict(X)
y = pd.DataFrame(y).transpose()
y_array = pd.DataFrame.as_matrix(y)
i = y_array.item(0)
j = y_array.item(1)
k = y_array.item(2)
output = (i, j, k)
return i, j, k
At the first iteration the program works fine, but at the second iteration I get an unhandled exception at line:
pModule = PyImport_Import(pName);
And in particular:
Unhandled exception at 0x00000001800db3ec in Fmain.exe: 0xC0000005: Access
violation writing location 0x0000000000000002.
Why is this happening? I tried to reload the python module after the first iteration, but the same thing happened. Any suggestions or other comments on this are much appreciated.
The goal is to create a numpy array in c++ and access it in python.
The block of code below runs fine when run as a program.
However, if I use ctypes and run the function, it segfaults on _import_array call. Could someone tell me why this happens ?
#include "Python.h"
#include "numpy/arrayobject.h"
using namespace std;
extern "C"
PyObject* test_create_numpy() {
Py_Initialize();
_import_array();
double* a = new double[1];
a[0] = 1.0;
npy_intp dims = {1};
PyObject *Arr = PyArray_SimpleNewFromData( 1, &dims, PyArray_FLOAT64, a);
return Arr;
}
int main() {
test_create_numpy();
return 0;
}
The python code used:
utils = cdll.LoadLibrary("test.dylib")
test_create_numpy = utils.test_create_numpy
test_create_numpy.restype = py_object
ret_vec = test_create_numpy()
print ret_vec
I'm trying to load a python module that contains a math and numpy import in C, using the C API. I can load and run the module but, if I import the math module it doesn't work.
I'm using Arch Linux, Python 2.7.2 and gcc.
Here the codes:
#include <stdio.h>
#include <stdlib.h>
#include <python2.7/Python.h>
int main(int argc, char **argv)
{
PyObject *pName, *pModule, *pFunc, *pArg, *pDict, *pReturn, *pT1, *pT2, *pX, *pY;
int i;
double x, y;
Py_Initialize();
PySys_SetPath(".");
pName = PyString_FromString("func");
if (!pName)
{
printf("pName\n");
return 0;
}
pModule = PyImport_Import(pName);
pDict = PyModule_GetDict(pModule);
pFunc = PyDict_GetItemString(pDict, "get_vals");
pArg = PyTuple_New(2);
PyTuple_SetItem(pArg, 0, PyFloat_FromDouble(4.0));
PyTuple_SetItem(pArg, 1, PyFloat_FromDouble(2.0));
pReturn = PyObject_CallObject(pFunc, pArg);
pT1 = PyTuple_GetItem(pReturn, 0);
pT2 = PyTuple_GetItem(pReturn, 1);
for (i = 0; i < PyTuple_Size(pT1); i++)
{
pX = PyTuple_GetItem(pT1, i);
pY = PyTuple_GetItem(pT2, i);
x = PyFloat_AsDouble(pX);
y = PyFloat_AsDouble(pY);
Py_XDECREF(pX);
Py_XDECREF(pY);
pX = NULL;
pY = NULL;
printf("Point p position is: %.2fx, %.2fy", x, y);
}
Py_XDECREF(pName); Py_XDECREF(pModule); Py_XDECREF(pFunc); Py_XDECREF(pArg); Py_XDECREF(pDict); Py_XDECREF(pReturn); Py_XDECREF(pT1); Py_XDECREF(pT2);
Py_Finalize();
return 0;
}
func.py
from math import cos
def get_vals(width, height):
x = (1, 2)
y = (cos(3), 4)
return x, y
And how can I embbed the Python script to C without need to use the script?
The PySys_SetPath(".") cleared the python path, so it could no longer find any library whatsoever. What you really need to do is import sys.path and then append your string to it:
PyObject *sys = PyImport_ImportModule("sys");
PyObject *path = PyObject_GetAttrString(sys, "path");
PyList_Append(path, PyString_FromString("."));
(I didn't test the above code, but it should be close. Also, you should do error checking)