Develop Reference

How to pass a vector by reference in pybind11 & c++

I try to pass vector/array by reference from python through pybind11 to a C++ library. The C++ library may fill in data. After the call to C++, I hope the python side will get the data.
Here is the simplified C++ code:
#include <pybind11/pybind11.h>
#include <pybind11/numpy.h>
#include <pybind11/stl.h>
class Setup
{
public:
Setup(int version) : _version(version) {}
int _version;
};
class Calculator
{
public:
Calculator() {}
static void calc(const Setup& setup, std::vector<double>& results) { ... }
}
namespace py = pybind11;
PYBIND11_MODULE(one_calculator, m) {
// optional module docstring
m.doc() = "pybind11 one_calculator plugin";
py::class_<Setup>(m, "Setup")
.def(py::init<int>());
py::class_<Calculator>(m, "Calculator")
.def(py::init<>())
.def("calc", &Calculator::calc);
}
On the python side, I intend to:
import os
import sys
import numpy as np
import pandas as pd
sys.path.append(os.path.realpath('...'))
from one_calculator import Setup, Calculator
a_setup = Setup(1)
a_calculator = Calculator()
results = []
a_calculator.calc(a_setup, results)
results
Apparently the results are not passed back. Is there a neat way to do it?

Figured out a way:
#include <pybind11/pybind11.h>
#include <pybind11/numpy.h>
#include <pybind11/stl.h>
#include "Calculator.h" // where run_calculator is
namespace py = pybind11;
// wrap c++ function with Numpy array IO
int wrapper(const std::string& input_file, py::array_t<double>& in_results) {
if (in_results.ndim() != 2)
throw std::runtime_error("Results should be a 2-D Numpy array");
auto buf = in_results.request();
double* ptr = (double*)buf.ptr;
size_t N = in_results.shape()[0];
size_t M = in_results.shape()[1];
std::vector<std::vector<double> > results;
run_calculator(input_file, results);
size_t pos = 0;
for (size_t i = 0; i < results.size(); i++) {
const std::vector<double>& line_data = results[i];
for (size_t j = 0; j < line_data.size(); j++) {
ptr[pos] = line_data[j];
pos++;
}
}
}
PYBIND11_MODULE(calculator, m) {
// optional module docstring
m.doc() = "pybind11 calculator plugin";
m.def("run_calculator", &wrapper, "Run the calculator");
}
Python side
results= np.zeros((N, M))
run_calculator(input_file, results)
This way I also do not expose classes Setup and Calculator to the python side.

Unknown type name 'PyModuleDef'

I'm working with Python and C++ integration, and I'm on MacOS.
When I use all of the PyObject methods they all seem to work but none of the PyModule methods seem to be working.
This is my code:
#include <iostream>
#include <cmath>
#include <vector>
#if PY_MAJOR_VERSION >= 3
#define PY3K
#endif
#ifdef __APPLE__
#include <Python/Python.h>
#else
#include <Python.h>
#endif
#define space " "
#define epoch int(1000)
using namespace std;
double hypothesis(const std::vector<double>& b_val, double x){
return b_val[0] + b_val[1] * x;
}
std::vector<double> regression(const std::vector<double>& x, const std::vector<double>& y,
int epochs, double learning_rate){
if(!epochs) epochs = epoch;
double _m = 0;
double _b = 0;
std::vector<double> _values(2);
int N = x.size();
for(int i = 0; i < epochs; i++){
_values[0] = _b, _values[1] = _m;
double dm = 0;
double db = 0;
double cost = 0;
for(int j = 0; j < N; j++){
double p = hypothesis(_values, x[j]);
cost += pow(y[j] - p, 2);
dm += (-2.0 / N) * (x[j] * (y[j] - p));
db += (-2.0 / N) * (y[j] - p);
}
cost /= N;
_m = _m - (learning_rate * dm);
_b = _b - (learning_rate * db);
if ((i + 1) % 100 == 0)
std::cout << "Epoch: " << (i + 1) << " Cost: " << cost << std::endl;
}
std::vector<double> result(2);
result[0] = _m, result[1] = _b;
return result;
}
static PyObject * fit(PyObject * self, PyObject * args){
PyObject *x;
PyObject *y;
double learning_rate;
int epochs;
int N;
if(!PyArg_ParseTuple(args, "00di0i", &x, &y, &epochs, &learning_rate, &N)){
return nullptr;
}
std::vector<double> _x(N), _y(N);
for(int i = 0; i < N; i++){
_x[i] = PyFloat_AsDouble(PyList_GetItem(x, (Py_ssize_t)i));
_y[i] = PyFloat_AsDouble(PyList_GetItem(y, (Py_ssize_t)i));
}
std::vector<double> _result = regression(_x, _y, epochs, learning_rate);
PyObject *result = PyTuple_New(2);
for(int i = 0; i < 2; i++){
PyTuple_SetItem(result, i, PyFloat_FromDouble(_result[i]));
}
return Py_BuildValue("s", result);
}
static PyMethodDef linreg_methods[] = {
{"fit", (PyCFunction)fit, METH_VARARGS, "Linear Regression"},
{nullptr}
};
static PyModuleDef linear_regression = {
PyModuleDef_HEAD_INIT,
"linear_regression"
"Linear Regression",
-1,
linreg_methods
};
PyMODINIT_FUNC PyInit_linear_regression(void){
return PyModule_Create(&linear_regression);
}
The output I get is error: unknown type name 'PyModuleDef'.
I can't seem to understand what the problem is.

You will need to make sure the included Python.h is from the Python3.x build, and that you link to the corresponding library, for example, on Linux that would be:
g++ my_module.cpp -I/usr/include/python3.8/ -lpython3.8
But on MacOS this file would be dependent on where/how you installed Python3. You should be able to use locate Python.h and find the Python3 directory.

How to call a Python function that uses numpy from C++ code

I have the following Python module (actual module I want to use, has many more functions using numpy and tensorflow), but this is a representative example to reproduce the issue
import numpy as np
def get_random():
return np.random.random()
I want to call the function get_random from the following C++ program:
#include <Python.h>
#include <stdio.h>
#include <string.h>
int main(int argc, char *argv[]){
Py_Initialize();
if( !Py_IsInitialized() ){
printf("Initialize failed\n");
return -1;
}
PyRun_SimpleString("import sys");
//append path of numpy lib
PyRun_SimpleString("sys.path.append('/home/uji300/.pyenv/versions/venv/lib/python3.4/site-packages/numpy/core/include')");
PyObject *pName, *pModule, *pDict, *pFunc, *pArgs, *pRet;
pName = PyUnicode_DecodeFSDefault("numpytester");
pModule = PyImport_Import(pName);
if ( !pModule ){
return -1;
}
pDict = PyModule_GetDict(pModule);
if ( !pDict ){
return -1;
}
pFunc = PyDict_GetItemString(pDict, "get_random");
if ( !pFunc || !PyCallable_Check(pFunc) ){
return -1;
}
for( int i = 0; i < 5; ++i ){
printf(" ===========> START CALL PYTHON SCRIPT %d <===========\n", i + 1);
pRet = PyObject_CallObject(pFunc, NULL); // call the function
printf(" ===========> CALLING FINISHED %d <===========\n", i + 1);
double result = PyFloat_AsDouble(pRet); // get the return value by pRet
printf(" ===========> result = %f <===========\n", result);
}
Py_DECREF(pName);
Py_DECREF(pModule);
Py_DECREF(pArgs);
Py_DECREF(pRet);
Py_DECREF(pDict);
// close Python
Py_Finalize();
return 0;
}
The C++ program is then built with CMake and run. The call to the get_random() does not work. It returns -1 every time.
What is the right way to call a python function that uses numpy or tensorflow functions ?

Problems in reloading python module from C++ static library being used in Fortran project

I'm trying to link Fortran|C/C++|Python using VS2010 on Windows 64-bit. I have a main code that is written in Fortran. From that code I call a C++ function and after that I call a Python function to do some staff further. Here is a simplified version of my code:
!Fmain.f90
PROGRAM fmain
IMPLICIT NONE
INTERFACE
SUBROUTINE Call_NN(input_array, output_array) BIND(C, name='Call_NN')
USE, INTRINSIC :: iso_c_binding
IMPLICIT NONE
REAL(C_DOUBLE), INTENT(IN), DIMENSION(*) :: input_array
REAL(C_DOUBLE), INTENT(INOUT), DIMENSION(*) :: output_array
END SUBROUTINE
END INTERFACE
REAL*8, DIMENSION(0:2) :: input_array, output_array
REAL :: b
INTEGER :: i
do i = 1, 3
input_array = 0.01d0
output_array = 0.d0
call Call_NN(input_array, output_array)
enddo
END
The C++ static library is as following:
//Csub.cpp
#include <Python.h>
#include <cassert>
#include <stdio.h>
#include <conio.h>
#include <iostream>
#include <fstream>
#include <array>
#include <string.h>
#include <errno.h>
#include <limits.h>
#include <assert.h>
#include <stdlib.h>
using namespace std;
extern "C" void Call_NN(array<double,3> input_array, array<double,3> output_array)
{
// Initialize the Python interpreter.
Py_Initialize();
// Variables Declaration
float result = 0;
float result1 = 0;
float result2 = 0;
float result3 = 0;
float value1 = 0;
float value2 = 0;
float value3 = 0;
// Create some Python objects that will later be assigned values.
PyObject *pName, *pModule, *pDict, *pFunc, *pArgs, *pValue1, *pValue2, *pValue3;
PyObject *pT1, *pT2, *pT3;
// Convert the file name to a Python string.
const char* filename = "module";
pName = PyString_FromString(filename);
if (pName == nullptr)
{
PyErr_Print();
std::exit(1);
}
// Import the file as a Python module.
pModule = PyImport_Import(pName);
if (pModule == nullptr)
{
PyErr_Print();
std::exit(1);
}
// Create a dictionary for the contents of the module.
pDict = PyModule_GetDict(pModule);
if (pDict == nullptr)
{
PyErr_Print();
std::exit(1);
}
// Get the add method from the dictionary.
pFunc = PyDict_GetItemString(pDict, "NN");
if (pFunc == nullptr)
{
PyErr_Print();
std::exit(1);
}
// Create a Python tuple to hold the arguments to the method.
pArgs = PyTuple_New(3);
if (pArgs == nullptr)
{
PyErr_Print();
std::exit(1);
}
// Convert 3 to a Python integer.
value1 = input_array[0];
value2 = input_array[1];
value3 = input_array[2];
pValue1 = PyFloat_FromDouble(value1);
pValue2 = PyFloat_FromDouble(value2);
pValue3 = PyFloat_FromDouble(value3);
// Set the Python int as the first and second arguments to the method.
PyTuple_SetItem(pArgs, 0, pValue1);
PyTuple_SetItem(pArgs, 1, pValue2);
PyTuple_SetItem(pArgs, 2, pValue3);
// Call the function with the arguments.
PyObject* pResult = PyObject_CallObject(pFunc, pArgs);
// Print a message if calling the method failed.
if (pResult == NULL)
printf("Calling the add method failed.\n");
// Convert the result to a long from a Python object.
//result = PyFloat_AsDouble(pResult);
pT1 = PyTuple_GetItem(pResult, 0);
pT2 = PyTuple_GetItem(pResult, 1);
pT3 = PyTuple_GetItem(pResult, 2);
// Convert output to float
result1 = PyFloat_AsDouble(pT1);
result2 = PyFloat_AsDouble(pT2);
result3 = PyFloat_AsDouble(pT3);
output_array[0] = result1;
output_array[1] = result2;
output_array[2] = result3;
// Destroy the Python interpreter.
Py_Finalize();
}
And the Python module is defined as:
# module.py
import sys
import numpy as np
import pandas as pd
import pickle
from sklearn.externals import joblib
def NN(a,b,c,):
X_array = np.array([a, b, c])
X = pd.DataFrame(X_array).transpose()
clf = joblib.load('SavedNeuralNetwork.pkl')
y = clf.predict(X)
y = pd.DataFrame(y).transpose()
y_array = pd.DataFrame.as_matrix(y)
i = y_array.item(0)
j = y_array.item(1)
k = y_array.item(2)
output = (i, j, k)
return i, j, k
At the first iteration the program works fine, but at the second iteration I get an unhandled exception at line:
pModule = PyImport_Import(pName);
And in particular:
Unhandled exception at 0x00000001800db3ec in Fmain.exe: 0xC0000005: Access
violation writing location 0x0000000000000002.
Why is this happening? I tried to reload the python module after the first iteration, but the same thing happened. Any suggestions or other comments on this are much appreciated.

Passing two parameters (int and array) to embedded Python function

I need to call Python function from my module and set two parameters for it: int and array.
For a now I get segfault during calling this function and I have no idea what I'm doing wrong. Could someone specify where my mistake is?
Function in my Python module app.py. It works if I call it from Python code:
def get_model(rate, signal):
mfcc_train = MFCC().compute(rate, signal)
with open('mfcc_test', 'wb') as f:
pickle.dump(mfcc_train, f)
return clf()._fit(mfcc_train)
My C code that calls the function above. The last pring is "Before calling"
#include <Python.h>
#include <stdio.h>
#include "wav.h"
#include <numpy/arrayobject.h>
int main(int argc, char *argv[])
{
PyObject *pName, *pModule, *pDict, *pFunc, *pValue, *pArgs;
uint8_t *samples = NULL;
wavread("test.wav", &samples);
printf("No. of channels: %d\n", header->num_channels);
printf("Sample rate: %d\n", header->sample_rate);
printf("Bit rate: %dkbps\n", header->byte_rate*8 / 1000);
printf("Bits per sample: %d\n\n", header->bps);
printf("Sample 0: %d\n", samples[0]);
printf("Sample 1: %d\n", samples[1]);
// Initialize the Python Interpreter
printf("Before init\n");
Py_Initialize();
PyObject *sysPath = PySys_GetObject("path");
const char *scriptDirectoryName = ".";
PyObject *path = PyUnicode_FromString(scriptDirectoryName);
int result = PyList_Insert(sysPath, 0, path);
printf("after init\n");
// Build the name object
pName = PyUnicode_DecodeFSDefault(argv[1]);
printf("after pname %s %d\n", argv[1], pName == NULL ? 1 : 0);
// Load the module object
pModule = PyImport_Import(pName);
printf("after pmodule %d\n", pModule == NULL ? 1 : 0);
// pFunc is also a borrowed reference
pFunc = PyObject_GetAttrString(pModule, "get_model");
printf("after pfunc\n");
if (PyCallable_Check(pFunc))
{
pArgs = PyTuple_New(2);
printf("after pytuple\n");
PyTuple_SetItem(pArgs, 0, PyLong_FromLong(header->sample_rate));
printf("after set item\n");
uint8_t* array = malloc(header->datachunk_size);
int dims[1];
dims[0] = header->datachunk_size;
printf("alloc\n");
import_array();
PyObject* pSamples = PyArray_SimpleNewFromData(1, dims, NPY_INT8, (void*)samples);
printf("pSamples\n");
PyArray_ENABLEFLAGS((PyArrayObject*)pSamples, NPY_ARRAY_OWNDATA);
PyTuple_SetItem(pArgs, 1, pSamples);
printf("Before calling\n");
pValue = PyObject_CallObject(pFunc, pArgs);
printf("After calling\n");
} else
{
PyErr_Print();
}
printf("pValue: %d\n", pValue);
// Clean up
Py_DECREF(pModule);
Py_DECREF(pFunc);
Py_DECREF(pName);
// Finish the Python Interpreter
Py_Finalize();
free(header);
free(samples);
}
UPD: updated code where one issue was fixed. But another problem still exists. It's in line PyObject* pSamples = PyArray_SimpleNewFromData(1, dims, NPY_INT8, (void*)samples);. And I can't find out what is wrong with it.
And wav.h just in case:
#include <inttypes.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <err.h>
typedef struct {
char chunk_id[4];
uint32_t chunk_size;
char format[4];
char fmtchunk_id[4];
uint32_t fmtchunk_size;
uint16_t audio_format;
uint16_t num_channels;
uint32_t sample_rate;
uint32_t byte_rate;
uint16_t block_align;
uint16_t bps;
char datachunk_id[4];
uint32_t datachunk_size;
}WavHeader;
WavHeader *header;
void wavread(char *file_name, int16_t **samples)
{
int fd;
if (!file_name)
errx(1, "Filename not specified");
if ((fd = open(file_name, O_RDONLY)) < 1)
errx(1, "Error opening file");
if (!header)
header = (WavHeader*)malloc(sizeof(WavHeader));
if (read(fd, header, sizeof(WavHeader)) < sizeof(WavHeader))
errx(1, "File broken: header");
if (strncmp(header->chunk_id, "RIFF", 4) ||
strncmp(header->format, "WAVE", 4))
errx(1, "Not a wav file");
if (header->audio_format != 1)
errx(1, "Only PCM encoding supported");
if (*samples) free(*samples);
*samples = (int16_t*)malloc(header->datachunk_size);
if (!*samples)
errx(1, "Error allocating memory");
if (read(fd, *samples, header->datachunk_size) < header->datachunk_size)
errx(1, "File broken: samples");
close(fd);
}

It's difficult to tell without the definition of header but I believe the issue is in the line
PyTuple_SetItem(pArgs, 0, header->sample_rate);
PyTuple_SetItem expects a Python object and you're passing it what I think is an integer, which is being misinterpreted as a PyObject*.
I suspect you want
PyTuple_SetItem(pArgs, 0, PyInt_FromLong(header->sample_rate));
(PyLong_FromLong in Python3)
Second issue: you free samples twice. First you pass it to numpy and tell numpy that it owns the data:
PyObject* pSamples = PyArray_SimpleNewFromData(1, dims, NPY_INT8, (void*)samples);
PyArray_ENABLEFLAGS((PyArrayObject*)pSamples, NPY_ARRAY_OWNDATA);
then at the end of your code you free it
free(samples);
I suspect that you meant to pass your newly allocated array to numpy instead of samples. (You still need to copy the data between them too, if this is the case)
UPD: One more right solution from comments is to change type of dims from int to npy_intp