From the official docs ;
#define PY_SSIZE_T_CLEAN
#include <Python.h>
int main(int argc, char *argv[])
{
wchar_t *program = Py_DecodeLocale(argv[0], NULL);
if (program == NULL) {
fprintf(stderr, "Fatal error: cannot decode argv[0]\n");
exit(1);
}
Py_SetProgramName(program); /* optional but recommended */
Py_Initialize();
PyRun_SimpleString("from time import time,ctime\n"
"print('Today is', ctime(time()))\n");
if (Py_FinalizeEx() < 0) {
exit(120);
}
PyMem_RawFree(program);
return 0;
}
I successfully ran this piece. I am trying to get output of PyRun_SimpleString to a string variable, let's say
char string[50];
I went through the documentation but couldn't come up with a result. As far as I can tell there are multiple ways to achieve this tiny task and is in the orders of a couple of additional lines. I would appreciate a guidence or a workaround regarding this manner.
I wish I had found a better way, but this way seems to work, (will update if I find a better solution):
If you defined a class in Python to catch sys.stdout writes:
import sys
class CatchOutErr:
def __init__(self):
self.value = ''
def write(self, txt):
self.value += txt
catchOutErr = CatchOutErr()
sys.stdout = catchOutErr
sys.stderr = catchOutErr
And you passed the value received from this handler to C++, converted to string, etc...
#include <Python.h>
#include <iostream>
#include <stdio.h>
int main(int argc, char *argv[])
{
Py_Initialize();
PyObject *pModule = PyImport_AddModule("__main__"); //create main module
std::string stdOutErr = "import sys\nclass CatchOutErr:\n\tdef __init__(self):\n\t\tself.value = ''\n\tdef write(self, txt):\n\t\tself.value += txt\ncatchOutErr = CatchOutErr()\nsys.stdout = catchOutErr\nsys.stderr = catchOutErr\n";
PyRun_SimpleString(stdOutErr.c_str()); //invoke code to redirect
PyRun_SimpleString("from time import time,ctime\n"
"print('Today is', ctime(time()))\n");
PyObject *catcher = PyObject_GetAttrString(pModule, "catchOutErr"); //get our catchOutErr created above
PyObject *output = PyObject_GetAttrString(catcher,"value"); //get the stdout and stderr from our catchOutErr object
std::string s = PyString_AsString(output);
Py_DECREF(catcher);
Py_DECREF(output);
Py_DECREF(s);
std::cout << s;
return 0;
}
('Today is', 'Thu Jul 30 09:02:55 2020')
For Python3:
...
PyObject *output = PyObject_GetAttrString(catcher,"value"); //get the stdout and stderr from our catchOutErr object
PyObject *encodedData = PyUnicode_AsEncodedString(output, "ascii", NULL);
Py_DECREF(output);
Py_DECREF(encodedData);
char* buf;
Py_ssize_t len;
PyBytes_AsStringAndSize(encodedData, &buf, &len);
std::cout << std::string(buf) << std::endl;
Reference:
https://stackoverflow.com/a/4307737/9238288
https://cpp.hotexamples.com/examples/-/-/PyUnicode_AsEncodedString/cpp-pyunicode_asencodedstring-function-examples.html
Related
I have the following Python module (actual module I want to use, has many more functions using numpy and tensorflow), but this is a representative example to reproduce the issue
import numpy as np
def get_random():
return np.random.random()
I want to call the function get_random from the following C++ program:
#include <Python.h>
#include <stdio.h>
#include <string.h>
int main(int argc, char *argv[]){
Py_Initialize();
if( !Py_IsInitialized() ){
printf("Initialize failed\n");
return -1;
}
PyRun_SimpleString("import sys");
//append path of numpy lib
PyRun_SimpleString("sys.path.append('/home/uji300/.pyenv/versions/venv/lib/python3.4/site-packages/numpy/core/include')");
PyObject *pName, *pModule, *pDict, *pFunc, *pArgs, *pRet;
pName = PyUnicode_DecodeFSDefault("numpytester");
pModule = PyImport_Import(pName);
if ( !pModule ){
return -1;
}
pDict = PyModule_GetDict(pModule);
if ( !pDict ){
return -1;
}
pFunc = PyDict_GetItemString(pDict, "get_random");
if ( !pFunc || !PyCallable_Check(pFunc) ){
return -1;
}
for( int i = 0; i < 5; ++i ){
printf(" ===========> START CALL PYTHON SCRIPT %d <===========\n", i + 1);
pRet = PyObject_CallObject(pFunc, NULL); // call the function
printf(" ===========> CALLING FINISHED %d <===========\n", i + 1);
double result = PyFloat_AsDouble(pRet); // get the return value by pRet
printf(" ===========> result = %f <===========\n", result);
}
Py_DECREF(pName);
Py_DECREF(pModule);
Py_DECREF(pArgs);
Py_DECREF(pRet);
Py_DECREF(pDict);
// close Python
Py_Finalize();
return 0;
}
The C++ program is then built with CMake and run. The call to the get_random() does not work. It returns -1 every time.
What is the right way to call a python function that uses numpy or tensorflow functions ?
I am trying to implement a Python wrapper using the Python C API over a C++ library. I need to implement conversions so I can use objects in Python and C++. I already done that in the past but I have an error I really have a hard time with.
I have a very basic test function:
PyObject* convert_to_python() {
std::cout << "Convert to PyObject" << std::endl;
long int a = 20;
PyObject* py_a = PyInt_FromLong(a);
std::cout << "Convert to PyObject ok" << std::endl;
return py_a;
}
I call this function inside a GoogleTest macro:
TEST(Wrapper, ConvertTest) {
PyObject *py_m = convert_to_python();
}
And my output is:
Convert to PyObject
Segmentation fault (core dumped)
I also ran valgrind on it:
valgrind --tool=memcheck --track-origins=yes --leak-check=full ./my_convert
But it doesn't give me much information about it:
Invalid read of size 8
==19030== at 0x4F70A7B: PyInt_FromLong (in /usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0)
==19030== by 0x541E6BF: _object* pysmud_from<float>(smu::Matrix<float, 0, 0>&) (smu_type_conversions.cpp:308)
==19030== by 0x43A144: (anonymous namespace)::Wrapper_ConvertMatrix_Test::Body() (test_wrapper.cpp:12)
==19030== by 0x43A0C6: (anonymous namespace)::Wrapper_ConvertMatrix_Test::TestBody() (test_wrapper.cpp:10)
==19030== by 0x465B4D: void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (gtest.cc:2078)
==19030== by 0x460684: void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (gtest.cc:2114)
==19030== by 0x444C05: testing::Test::Run() (gtest.cc:2151)
==19030== by 0x4454C9: testing::TestInfo::Run() (gtest.cc:2326)
==19030== by 0x445BEA: testing::TestCase::Run() (gtest.cc:2444)
==19030== by 0x44CF41: testing::internal::UnitTestImpl::RunAllTests() (gtest.cc:4315)
==19030== by 0x46712C: bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) (gtest.cc:2078)
==19030== by 0x461532: bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) (gtest.cc:2114)
==19030== Address 0x0 is not stack'd, malloc'd or (recently) free'd
I think this code should work but I can't get what it's wrong with what I wrote. Did I wrongly included or linked Python files and libraries ?
EDIT: Gives no errors
#include <Python.h>
PyObject* convert_long_int(long int a) {
PyObject *ret = PyInt_FromLong(a);
return ret;
}
int main(void) {
long int a = 65454984;
PyObject *pya = convert_long_int(a);
return 0;
}
If compiling with gcc -o wraptest -I/usr/include/python2.7 wraptest.c -L/usr/lib/x86_64-linux-gnu/ -lpython2.7
What does the initialization do ?
I can confirm the segmentation fault on Ubuntu 16.04 and Python 2.7, if I omit the initialization.
Looking at Embedding Python in Another Application, there's this example
#include <Python.h>
int
main(int argc, char *argv[])
{
Py_SetProgramName(argv[0]); /* optional but recommended */
Py_Initialize();
PyRun_SimpleString("from time import time,ctime\n"
"print 'Today is',ctime(time())\n");
Py_Finalize();
return 0;
}
So when I do an equivalent minimal main
int main()
{
Py_Initialize();
PyObject *p = convert_to_python();
Py_Finalize();
return 0;
}
it works without crash.
The difference between the two examples is
long int a = 20;
and
long int a = 65454984;
I guess, it has to do with PyInt_FromLong(long ival)
The current implementation keeps an array of integer objects for all integers between -5 and 256, when you create an int in that range you actually just get back a reference to the existing object.
Maybe Python tries to access an uninitialized pointer or memory range without the initialization.
When I change the example using a = 256, it crashes. Using a = 257, it doesn't.
Looking at cpython/Objects/intobject.c:79, you can see an array of pointers
static PyIntObject *small_ints[NSMALLNEGINTS + NSMALLPOSINTS];
which is accessed right below in PyInt_FromLong(long ival)
v = small_ints[ival + NSMALLNEGINTS];
Py_INCREF(v);
But without initialization from _PyInt_Init(void)
for (ival = -NSMALLNEGINTS; ival < NSMALLPOSINTS; ival++) {
if (!free_list && (free_list = fill_free_list()) == NULL)
return 0;
/* PyObject_New is inlined */
v = free_list;
free_list = (PyIntObject *)Py_TYPE(v);
(void)PyObject_INIT(v, &PyInt_Type);
v->ob_ival = ival;
small_ints[ival + NSMALLNEGINTS] = v;
}
these pointers are all NULL, causing the crash.
I've followed all of the basic steps trying to get a python module loaded in c++, however it seems that when I try to get the dictionary of items in the script, it ignores my functions and globals that I wanted to use inside. when I iterate through the items, all I get are the builtins, file, package, path, name, and doc attributes of the script, and nothing else. I checked __name__ and it is coming up correctly ("test.py" is my py file's name and it returns "test" just fine). When I actually try to load my functions or globals (test, qa), the PyDict_GetItemString function returns NULL. What have I done wrong such that in tutorials this works fine, but in my test application, it doesn't work?
here is my Py script, maybe I've forgotten to do something that would allow my items to be seen?
qa = "hello test"
def test(a):
q = "hello world, I am " + a
#print q
return q
here is my C++ code as well, maybe I've forgotten something here?
#include <iostream>
#include <Python.h>
int main() {
Py_Initialize();
PyObject
*pName,
*pModule,
*pDict,
*pFunc,
*pArgs,
*pValue;
// get filename as a pystring
pName = PyString_FromString("test");
std::cout << std::endl << pName;
// Import module from filename
pModule = PyImport_Import(pName);
std::cout << std::endl << pModule;
// build the module's dict
pDict = PyModule_GetDict(pModule);
std::cout << std::endl << pDict << " " << PyDict_Size(pDict);
PyObject* keys = PyDict_Keys(pDict);
int s = PyList_Size(keys);
for (int i = 0; i < s; ++i) {
PyObject* item = PyList_GetItem(keys, i);
printf("\n");
printf(PyString_AsString(item));
}
PyObject* testvar = PyDict_GetItemString(pDict, "qa");
printf(PyString_AsString(testvar));
// get a function from the dict
pFunc = PyDict_GetItemString(pDict, "test");
std::cout << std::endl << pFunc;
// build the arg tuple
pArgs = PyTuple_New(1);
// create an argument
pValue = PyString_FromString("cee programme");
// set an argument
PyTuple_SetItem(pArgs, 0, pValue);
// call the function with the func and the args
PyObject* pResult = PyObject_CallObject(pFunc, pArgs);
// error checking
if (pResult == NULL) {
printf("\nis broek");
}
char* res = PyString_AsString(pResult);
// "destroy the interpreter"
Py_Finalize();
printf(res);
return 0;
}
I need to call Python function from my module and set two parameters for it: int and array.
For a now I get segfault during calling this function and I have no idea what I'm doing wrong. Could someone specify where my mistake is?
Function in my Python module app.py. It works if I call it from Python code:
def get_model(rate, signal):
mfcc_train = MFCC().compute(rate, signal)
with open('mfcc_test', 'wb') as f:
pickle.dump(mfcc_train, f)
return clf()._fit(mfcc_train)
My C code that calls the function above. The last pring is "Before calling"
#include <Python.h>
#include <stdio.h>
#include "wav.h"
#include <numpy/arrayobject.h>
int main(int argc, char *argv[])
{
PyObject *pName, *pModule, *pDict, *pFunc, *pValue, *pArgs;
uint8_t *samples = NULL;
wavread("test.wav", &samples);
printf("No. of channels: %d\n", header->num_channels);
printf("Sample rate: %d\n", header->sample_rate);
printf("Bit rate: %dkbps\n", header->byte_rate*8 / 1000);
printf("Bits per sample: %d\n\n", header->bps);
printf("Sample 0: %d\n", samples[0]);
printf("Sample 1: %d\n", samples[1]);
// Initialize the Python Interpreter
printf("Before init\n");
Py_Initialize();
PyObject *sysPath = PySys_GetObject("path");
const char *scriptDirectoryName = ".";
PyObject *path = PyUnicode_FromString(scriptDirectoryName);
int result = PyList_Insert(sysPath, 0, path);
printf("after init\n");
// Build the name object
pName = PyUnicode_DecodeFSDefault(argv[1]);
printf("after pname %s %d\n", argv[1], pName == NULL ? 1 : 0);
// Load the module object
pModule = PyImport_Import(pName);
printf("after pmodule %d\n", pModule == NULL ? 1 : 0);
// pFunc is also a borrowed reference
pFunc = PyObject_GetAttrString(pModule, "get_model");
printf("after pfunc\n");
if (PyCallable_Check(pFunc))
{
pArgs = PyTuple_New(2);
printf("after pytuple\n");
PyTuple_SetItem(pArgs, 0, PyLong_FromLong(header->sample_rate));
printf("after set item\n");
uint8_t* array = malloc(header->datachunk_size);
int dims[1];
dims[0] = header->datachunk_size;
printf("alloc\n");
import_array();
PyObject* pSamples = PyArray_SimpleNewFromData(1, dims, NPY_INT8, (void*)samples);
printf("pSamples\n");
PyArray_ENABLEFLAGS((PyArrayObject*)pSamples, NPY_ARRAY_OWNDATA);
PyTuple_SetItem(pArgs, 1, pSamples);
printf("Before calling\n");
pValue = PyObject_CallObject(pFunc, pArgs);
printf("After calling\n");
} else
{
PyErr_Print();
}
printf("pValue: %d\n", pValue);
// Clean up
Py_DECREF(pModule);
Py_DECREF(pFunc);
Py_DECREF(pName);
// Finish the Python Interpreter
Py_Finalize();
free(header);
free(samples);
}
UPD: updated code where one issue was fixed. But another problem still exists. It's in line PyObject* pSamples = PyArray_SimpleNewFromData(1, dims, NPY_INT8, (void*)samples);. And I can't find out what is wrong with it.
And wav.h just in case:
#include <inttypes.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <err.h>
typedef struct {
char chunk_id[4];
uint32_t chunk_size;
char format[4];
char fmtchunk_id[4];
uint32_t fmtchunk_size;
uint16_t audio_format;
uint16_t num_channels;
uint32_t sample_rate;
uint32_t byte_rate;
uint16_t block_align;
uint16_t bps;
char datachunk_id[4];
uint32_t datachunk_size;
}WavHeader;
WavHeader *header;
void wavread(char *file_name, int16_t **samples)
{
int fd;
if (!file_name)
errx(1, "Filename not specified");
if ((fd = open(file_name, O_RDONLY)) < 1)
errx(1, "Error opening file");
if (!header)
header = (WavHeader*)malloc(sizeof(WavHeader));
if (read(fd, header, sizeof(WavHeader)) < sizeof(WavHeader))
errx(1, "File broken: header");
if (strncmp(header->chunk_id, "RIFF", 4) ||
strncmp(header->format, "WAVE", 4))
errx(1, "Not a wav file");
if (header->audio_format != 1)
errx(1, "Only PCM encoding supported");
if (*samples) free(*samples);
*samples = (int16_t*)malloc(header->datachunk_size);
if (!*samples)
errx(1, "Error allocating memory");
if (read(fd, *samples, header->datachunk_size) < header->datachunk_size)
errx(1, "File broken: samples");
close(fd);
}
It's difficult to tell without the definition of header but I believe the issue is in the line
PyTuple_SetItem(pArgs, 0, header->sample_rate);
PyTuple_SetItem expects a Python object and you're passing it what I think is an integer, which is being misinterpreted as a PyObject*.
I suspect you want
PyTuple_SetItem(pArgs, 0, PyInt_FromLong(header->sample_rate));
(PyLong_FromLong in Python3)
Second issue: you free samples twice. First you pass it to numpy and tell numpy that it owns the data:
PyObject* pSamples = PyArray_SimpleNewFromData(1, dims, NPY_INT8, (void*)samples);
PyArray_ENABLEFLAGS((PyArrayObject*)pSamples, NPY_ARRAY_OWNDATA);
then at the end of your code you free it
free(samples);
I suspect that you meant to pass your newly allocated array to numpy instead of samples. (You still need to copy the data between them too, if this is the case)
UPD: One more right solution from comments is to change type of dims from int to npy_intp
For my project (ia for a game) I need a C network interface. I want to code my project in Python so I created a C++ wrapper from C TCP-Socket functions, to use it with c-types in my Python code:
#include <iostream>
#include <string>
#include "Socket.h"
#include "stdio.h"
void Socket::s_connect(char *host, int port){
const char *str;
pe = getprotobyname("tcp");
if (!pe){
perror("Error get protocol");
exit(1);
}
f = socket(AF_INET, SOCK_STREAM, pe->p_proto);
if (f == -1){
perror("Error create socker");
exit(1);
}
s_in.sin_family = AF_INET;
s_in.sin_port = htons(port);
s_in.sin_addr.s_addr = inet_addr(host);
if (connect(f, (const struct sockaddr *&)s_in, sizeof(s_in)) == -1){
perror("Error on connect");
close(f);
}
}
void Socket::s_send(char *msg){
if (write(f, msg, strlen(msg)) == -1){
printf("error write\n");
perror("write");
exit(1);
}
}
char *Socket::s_recv(){
char *buff;
int ret;
buff = (char *)malloc(4096);
ret = read(f, buff, 4096);
if (ret < 0){
if (close(f))
exit(1);
}
return buff;
}
extern "C" {
Socket *Socket_new()
{
return (new Socket);
}
void Socket_connect(Socket *sock, char *host, int port)
{
sock->s_connect(host, port);
}
void Socket_send(Socket *sock, char *msg)
{
sock->s_send(msg);
}
}
Socket.h :
#ifndef SOCKET_HPP_
# define SOCKET_HPP_
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <errno.h>
class Socket {
int f, port;
struct protoent *pe;
struct sockaddr_in s_in;
char *ip;
public:
void s_connect(char *host, int port);
void s_send(char * msg);
char *s_recv();
};
#endif
And there is my Python class:
#!/usr/bin/python
import sys
from ctypes import cdll
lib = cdll.LoadLibrary('./libsocket.so')
class Socket(object):
def __init__(self):
self.obj = lib.Socket_new()
def s_connect(self, host, port):
print(host, port)
lib.Socket_connect(self.obj, host, int(port))
def s_send(self, msg):
lib.Socket_send(self.obj, msg)
def s_recv(self):
lib.Socket_recv(self.obj)
def main(arg):
sock = Socket()
sock.s_connect(arg[1], arg[2])
sock.s_send("coucou")
if __name__ == "__main__":
main(sys.argv)
And My extern C func can't read my string sent from Python I can send a port number but my C++ function can't read the host's string value.
If I change char * to std::string I get the same problem.
Did I do something wrong?
Thanks
As you say you use a Python 3 version, the documentation for the ctypes module states that Python unicode strings (str) correspond to wide character arrays in C (wchar_t *), and that Python byte strings (bytes) correspond to narrow character arrays (char *).
So depending if you want to actually process unicode characters, you should:
either use bytes Python side (trivial):
sock.s_connect(arg[1].encode('latin1'), arg[2])
or use wchar_t C side (probably harder...)