I'm trying to build a simple C extension for Python, initially I'm trying to return the contents of an unsigned char array (binary data).
unsigned char frame_out[200];
// test data
for (int i = 0; i < 199; i++) {
frame_out[i] = (i&0xff);
}
PyObject* result = Py_BuildValue("y#", frame_out, 199);
return result;
The code above works fine and I can print the test values in Python. But I want the array, frame_out, to be dynamic so I use new() as shown below.
char* frame_out = new char[200]
// test data
for (int i = 0; i < 199; i++) {
frame_out[i] = (i&0xff);
}
PyObject* result = Py_BuildValue("y#", frame_out, 199);
return result;
This now gives an error when called from Python:
Process finished with exit code -1073741819 (0xC0000005)
I have also tried using malloc:
char* frame_out = (char* )malloc( sizeof(char) * 200);
But this gives the same error.
I have checked result and this is not null. What have I done wrong?
Dev environment:
Python 3.6
Ubuntu 18.04.3 LTS
I'm currently writing a python wrapper for a shared library using ctypes.
The shared library is written in C++ and already has a C Wrapper:
#ifdef __cplusplus
extern "C" {
#endif //__cplusplus
#define UOM_EXPORT __attribute__((visibility("default")))
UOM_EXPORT MyModelInterface *createMyModel() {
return new MyModel();
}
UOM_EXPORT int C_setOption(MyModelInterface *m, const char *option, const char *value) {
if (m == NULL) {
return -1;
}
m->setOption(option, value);
return 0;
}
UOM_EXPORT int C_initializeModel(MyModelInterface *m, const char *inputFilename, float a1, float a2) {
if (m == NULL) {
return -1;
}
std::string input(inputFilename);
m->initializeModel(input, a1, a2);
return 0;
}
UOM_EXPORT int destroyMyModel(MyModelInterface *m) {
if (m == NULL) {
return -1;
}
delete reinterpret_cast <MyModel *>(m);
return 0;
}
#ifdef __cplusplus
}
#endif //__cplusplus
I created a class to act as my python interface to this C wrapper:
from ctypes import *
class c_model_interface(Structure):
def __init__(self):
self._buf
class MyModel:
library=None
model=None
def __init__(self,libraryPath):
self.library=CDLL(libraryPath)
exptr = POINTER(c_model_interface)
self.library.createModel.restype=(exptr)
self.library.createModel.argtypes=None
self.library.C_initializeModel.restype=c_int
self.library.C_initializeModel.argtypes=(exptr,c_char_p,c_float,c_float)
self.library.C_setOption.restype=c_int
self.library.C_setOption.argtypes=(exptr,c_char_p,c_char_p)
voidptr = self.library.createModel()
self.model = cast(voidptr,POINTER(c_model_interface))
def initializeModel(self,filename,a1,a2):
C_filename = bytes(filename,'utf8')
C_a1 = c_float(a1)
C_a2 = c_float(a2)
retV = self.library.C_initializeModel(self.model,C_filename,C_a1,C_a2)
if(retV != 0):
raise Exception("Exception in initializeModel")
def setOption(self,option,value):
if not isinstance(option,str):
raise Exception("Option needs to be a string")
if not isinstance(value,str):
raise Exception("Value needs to be a string")
C_option = cast(bytes(option,'utf8'),c_char_p)
C_value = cast(bytes(value,'utf8'),c_char_p)
retV = self.library.C_setOption(self.model,C_option,C_value)
if(retV != 0):
raise Exception("Exception in setOption")
def setOptions(self,optDict):
for strOpt,strValue in optDict.items():
self.setOption(strOpt,strValue)
It's then supposed to be called like:
libraryPath = "./libmymodel.so"
filename = "./test.png"
options = {"opt1":"val1","opt2":"val2"}
model = MyModel(libraryPath)
model.setOptions(options)
model.initializeModel(filename,1.0,2.0)
# ... more code
The code currently works. But whenever I make any mistake like typing:
def setOptions(self,optDict):
for strOpt,strValue in optDict: # optDict instead of optDict.items()
self.setOption(strOpt,strValue)
I get the general error:
free(): invalid pointer
Aborted (core dumped)
Instead of something specific like this for the dictionary error:
Traceback (most recent call last):
File "MyModel.py", line 102, in <module>
model.setOptions(options)
File "MyModel.py", line 64, in setOptions
for strOpt,strValue in optDict:
ValueError: too many values to unpack (expected 2)
This makes debugging a nightmare. Is there any way to fix this?
EDIT:
For a minimal example:
from ctypes import *
a = CDLL(mylib)
options = {"test1":1,"test2":2}
for a,b in options:
print(a,b)
This gives the free(): invalid pointer message, when using my own shared library as mylib. If I set mylib='libc.so.6' it correctly prints the Traceback.
EDIT2: changed the expected Traceback to be more realistic for the given example.
I'm trying to program with the python code which include one lib.so file. The C language had a callback method, wants me to put my string into given address.
I have spent a whole week to solve this problem....
Now, I've passed the string to C.
But the next problem arises... "C" can't free() my string pointer which create by python.
Error in `/usr/bin/python': free(): invalid pointer
I've omitted the other head code
typedef struct
{
int (*Info) (int resId,char **outbuff);
} OptFuncs;
here's the C code
OptFuncs g_sdk_opt_funcs = {NULL};
int SDK_INIT(OptFuncs *optfuncs)
{
g_sdk_opt_funcs = *optfuncs;
return 0;
}
int SDK_RUN()
{
resId = 6604;
char *szvalbuf = NULL;
g_sdk_opt_funcs.Info(resId,&szvalbuf);
if(szvalbuf) {free(szvalbuf); szvalbuf = NULL;}
// I guess that's the problem.
return 0;
}
here is the sample using C language:
int myInfo(int resId,char **outbuff)
{
int iret = 0;
*outbuff = NULL;
char buff[512];
int buflen = sizeof(buff);
memset(buff,0,sizeof(buff));
if(resId == 6604)
{
snprintf(buff,buflen,"4GB");
}
if(iret == 0)
{
*outbuff = (char*)malloc(strlen(buff)+1);
strcpy(*outbuff,buff);
}
return iret;
}
int main(int argc, char *argv[])
{
OptFuncs optfuncs={NULL};
optfuncs.Info = myInfo;
int ret = SDK_INIT(&optfuncs);
ret = SDK_RUN();
}
It works with pure C.
and my python function was:
lib = CDLL('./lib/lib.so')
infoCallback = CFUNCTYPE(c_int, c_int, POINTER(POINTER(c_char)))
class OptFuncs(Structure):
_fields_ = [("Info", infoCallback)]
def my_info(res_id, out_buff):
iret = 0
out_buff[0] = None
if res_id == 6604:
buff = '16GB'
char_array = create_string_buffer(buff)
out_buff[0] = cast(char_array, POINTER(c_char))
return iret
optFuncs = OptFuncs()
optFuncs.Info = infoCallback(my_info)
# initialize the lib‘s callback.
lib.SDK_INIT.argtypes = [POINTER(OptFuncs)]
ret = lib.SDK_INIT(pointer(optFuncs))
# run the lib‘s main func.
ret = lib.SDK_RUN()
And then the error happens.
Error in `/usr/bin/python': free(): invalid pointer
Did I do it wrong?
The problem is that memory is allocated by create_string_buffer in Python's C runtime library, and freed in the DLL's runtime library. They may not be compiled with the same version of compiler, and we don't know internally how create_string_buffer allocates the buffer. The pointer that the DLL receives may not be the pointer allocated. create_string_buffer may allocate more than you expect to store ctypes metadata. You don't want to be freeing memory managed by Python.
To fix this, ensure the data is allocated and freed in the DLL. For example add this function to the DLL and call it instead of create_string_buffer:
C
API char* SDK_MALLOC(const char* buff)
{
char* s = malloc(strlen(buff) + 1);
strcpy(s,buff);
return s;
}
Python
lib.SDK_MALLOC.argtypes = ()
lib.SDK_MALLOC.restype = POINTER(c_char)
my_info becomes:
def my_info(res_id, out_buff):
iret = 0
out_buff[0] = None
if res_id == 6604:
buff = b'16GB'
char_array = lib.SDK_MALLOC(buff)
out_buff.contents = char_array
return iret
I created embedded python interpreter which is calling a function, that has 3 parameters. I successfully managed to do this for function without parameters, but when I execute PyObject_CallObject the program crashes with SEGFAULT:
#0 0x00007ffff79c3a25 in PyEval_EvalCodeEx () from /usr/lib/libpython3.2mu.so.1.0
#1 0x00007ffff79c42bf in ?? () from /usr/lib/libpython3.2mu.so.1.0
#2 0x00007ffff79c730a in PyObject_Call () from /usr/lib/libpython3.2mu.so.1.0
#3 0x00000000004f3f31 in Huggle::Python::PythonScript::Hook_SpeedyFinished(Huggle::WikiEdit*, bool) ()
The source code of call is:
void PythonScript::Hook_SpeedyFinished(WikiEdit *edit, bool successfull)
{
if (edit == nullptr)
return;
if (this->ptr_Hook_SpeedyFinished != nullptr)
{
HUGGLE_DEBUG("Calling hook Hook_SpeedyFinished #" + this->Name, 2);
// let's make a new list of params
PyObject *args = PyTuple_New(3);
PyObject *page_name = PyUnicode_FromString(edit->Page->PageName.toUtf8().data());
PyObject *user_name = PyUnicode_FromString(edit->User->Username.toUtf8().data());
PyObject *success;
if (!successfull)
successfull = PyUnicode_FromString("fail");
else
successfull = PyUnicode_FromString("success");
if (PyTuple_SetItem(args, 0, page_name))
HUGGLE_DEBUG("Failed to pass page_name to tuple #hook_speedy_finished", 3);
if (PyTuple_SetItem(args, 1, user_name))
HUGGLE_DEBUG("Failed to pass user to tuple #hook_speedy_finished", 3);
if (PyTuple_SetItem(args, 2, success))
HUGGLE_DEBUG("Failed to pass success to tuple #hook_speedy_finished", 3);
PyObject_CallObject(this->ptr_Hook_SpeedyFinished, args);
HUGGLE_DEBUG("finished", 1);
}
}
Full source code of this .cpp file is https://github.com/huggle/huggle3-qt-lx/blob/master/huggle/pythonengine.cpp
What is wrong? Is this even proper way to call the function? I am following https://docs.python.org/3/c-api/object.html and https://docs.python.org/2/c-api/tuple.html
Here is an example from the python documentation.
PyObject *arglist;
...
arglist = Py_BuildValue("(l)", eventcode);
result = PyObject_CallObject(my_callback, arglist);
Py_DECREF(arglist);
if (result == NULL)
return NULL; /* Pass error back */
/* Here maybe use the result */
Py_DECREF(result);
It seems to me you should take care of the refcount.
https://docs.python.org/3.0/extending/extending.html#calling-python-functions-from-c
Segmentation fault is discussed here.
Calling python method from C++ (or C) callback
I'm having some trouble figuring out the proper way to walk a Python traceback using the C API. I'm writing an application that embeds the Python interpreter. I want to be able to execute arbitrary Python code, and if it raises an exception, to translate it to my own application-specific C++ exception. For now, it is sufficient to extract just the file name and line number where the Python exception was raised. This is what I have so far:
PyObject* pyresult = PyObject_CallObject(someCallablePythonObject, someArgs);
if (!pyresult)
{
PyObject* excType, *excValue, *excTraceback;
PyErr_Fetch(&excType, &excValue, &excTraceback);
PyErr_NormalizeException(&excType, &excValue, &excTraceback);
PyTracebackObject* traceback = (PyTracebackObject*)traceback;
// Advance to the last frame (python puts the most-recent call at the end)
while (traceback->tb_next != NULL)
traceback = traceback->tb_next;
// At this point I have access to the line number via traceback->tb_lineno,
// but where do I get the file name from?
// ...
}
Digging around in the Python source code, I see they access both the filename and module name of the current frame via the _frame structure, which looks like it is a privately-defined struct. My next idea was to programmatically load the Python 'traceback' module and call its functions with the C API. Is this sane? Is there a better way to access a Python traceback from C?
This is an old question but for future reference, you can get the current stack frame from the thread state object and then just walk the frames backward. A traceback object isn't necessary unless you want to preserve the state for the future.
For example:
PyThreadState *tstate = PyThreadState_GET();
if (NULL != tstate && NULL != tstate->frame) {
PyFrameObject *frame = tstate->frame;
printf("Python stack trace:\n");
while (NULL != frame) {
// int line = frame->f_lineno;
/*
frame->f_lineno will not always return the correct line number
you need to call PyCode_Addr2Line().
*/
int line = PyCode_Addr2Line(frame->f_code, frame->f_lasti);
const char *filename = PyString_AsString(frame->f_code->co_filename);
const char *funcname = PyString_AsString(frame->f_code->co_name);
printf(" %s(%d): %s\n", filename, line, funcname);
frame = frame->f_back;
}
}
I prefer calling into python from C:
err = PyErr_Occurred();
if (err != NULL) {
PyObject *ptype, *pvalue, *ptraceback;
PyObject *pystr, *module_name, *pyth_module, *pyth_func;
char *str;
PyErr_Fetch(&ptype, &pvalue, &ptraceback);
pystr = PyObject_Str(pvalue);
str = PyString_AsString(pystr);
error_description = strdup(str);
/* See if we can get a full traceback */
module_name = PyString_FromString("traceback");
pyth_module = PyImport_Import(module_name);
Py_DECREF(module_name);
if (pyth_module == NULL) {
full_backtrace = NULL;
return;
}
pyth_func = PyObject_GetAttrString(pyth_module, "format_exception");
if (pyth_func && PyCallable_Check(pyth_func)) {
PyObject *pyth_val;
pyth_val = PyObject_CallFunctionObjArgs(pyth_func, ptype, pvalue, ptraceback, NULL);
pystr = PyObject_Str(pyth_val);
str = PyString_AsString(pystr);
full_backtrace = strdup(str);
Py_DECREF(pyth_val);
}
}
I've discovered that _frame is actually defined in the frameobject.h header included with Python. Armed with this plus looking at traceback.c in the Python C implementation, we have:
#include <Python.h>
#include <frameobject.h>
PyTracebackObject* traceback = get_the_traceback();
int line = traceback->tb_lineno;
const char* filename = PyString_AsString(traceback->tb_frame->f_code->co_filename);
But this still seems really dirty to me.
One principal I've found useful in writing C extensions is to use each language where it's best suited. So if you have a task to do that would be best implemented in Python, implement in Python, and if it would be best implemented in C, do it in C. Interpreting tracebacks is best done in Python for two reasons: first, because Python has the tools to do it, and second, because it isn't speed-critical.
I would write a Python function to extract the info you need from the traceback, then call it from C.
You could even go so far as to write a Python wrapper for your callable execution. Instead of invoking someCallablePythonObject, pass it as an argument to your Python function:
def invokeSomeCallablePythonObject(obj, args):
try:
result = obj(*args)
ok = True
except:
# Do some mumbo-jumbo with the traceback, etc.
result = myTraceBackMunger(...)
ok = False
return ok, result
Then in your C code, call this Python function to do the work. The key here is to decide pragmatically which side of the C-Python split to put your code.
I used the following code to extract Python exception's error body. strExcType stores the exception type and strExcValue stores the exception body. Sample values are:
strExcType:"<class 'ImportError'>"
strExcValue:"ImportError("No module named 'nonexistingmodule'",)"
Cpp code:
if(PyErr_Occurred() != NULL) {
PyObject *pyExcType;
PyObject *pyExcValue;
PyObject *pyExcTraceback;
PyErr_Fetch(&pyExcType, &pyExcValue, &pyExcTraceback);
PyErr_NormalizeException(&pyExcType, &pyExcValue, &pyExcTraceback);
PyObject* str_exc_type = PyObject_Repr(pyExcType);
PyObject* pyStr = PyUnicode_AsEncodedString(str_exc_type, "utf-8", "Error ~");
const char *strExcType = PyBytes_AS_STRING(pyStr);
PyObject* str_exc_value = PyObject_Repr(pyExcValue);
PyObject* pyExcValueStr = PyUnicode_AsEncodedString(str_exc_value, "utf-8", "Error ~");
const char *strExcValue = PyBytes_AS_STRING(pyExcValueStr);
// When using PyErr_Restore() there is no need to use Py_XDECREF for these 3 pointers
//PyErr_Restore(pyExcType, pyExcValue, pyExcTraceback);
Py_XDECREF(pyExcType);
Py_XDECREF(pyExcValue);
Py_XDECREF(pyExcTraceback);
Py_XDECREF(str_exc_type);
Py_XDECREF(pyStr);
Py_XDECREF(str_exc_value);
Py_XDECREF(pyExcValueStr);
}
I had reason to do this recently while writing an allocation tracker for numpy. The previous answers are close but frame->f_lineno will not always return the correct line number--you need to call PyFrame_GetLineNumber(). Here's an updated code snippet:
#include "frameobject.h"
...
PyFrameObject* frame = PyEval_GetFrame();
int lineno = PyFrame_GetLineNumber(frame);
PyObject *filename = frame->f_code->co_filename;
The full thread state is also available in the PyFrameObject; if you want to walk the stack keep iterating on f_back until it's NULL. Checkout the full data structure in frameobject.h: http://svn.python.org/projects/python/trunk/Include/frameobject.h
See also: https://docs.python.org/2/c-api/reflection.html
You can access Python traceback similar to tb_printinternal function. It iterates over PyTracebackObject list. I have tried also suggestions above to iterate over frames, but it does not work for me (I see only the last stack frame).
Excerpts from CPython code:
static int
tb_displayline(PyObject *f, PyObject *filename, int lineno, PyObject *name)
{
int err;
PyObject *line;
if (filename == NULL || name == NULL)
return -1;
line = PyUnicode_FromFormat(" File \"%U\", line %d, in %U\n",
filename, lineno, name);
if (line == NULL)
return -1;
err = PyFile_WriteObject(line, f, Py_PRINT_RAW);
Py_DECREF(line);
if (err != 0)
return err;
/* ignore errors since we are not able to report them, are we? */
if (_Py_DisplaySourceLine(f, filename, lineno, 4))
PyErr_Clear();
return err;
}
static int
tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit)
{
int err = 0;
long depth = 0;
PyTracebackObject *tb1 = tb;
while (tb1 != NULL) {
depth++;
tb1 = tb1->tb_next;
}
while (tb != NULL && err == 0) {
if (depth <= limit) {
err = tb_displayline(f,
tb->tb_frame->f_code->co_filename,
tb->tb_lineno,
tb->tb_frame->f_code->co_name);
}
depth--;
tb = tb->tb_next;
if (err == 0)
err = PyErr_CheckSignals();
}
return err;
}
As of python 3.11, accessing the frame objects seems to need a different approach. Anyway, this works in 3.11, hth someone
py_err(void)
{
PyObject *err = PyErr_Occurred();
if (! err) {
return;
}
PyObject *ptype, *pvalue, *pbacktrace, *pyobj_str;
PyObject *ret, *list, *string;
PyObject *mod;
char *py_str;
PyErr_Fetch(&ptype, &pvalue, &pbacktrace);
PyErr_NormalizeException(&ptype, &pvalue, &pbacktrace);
PyErr_Display(ptype, pvalue, pbacktrace);
PyTraceBack_Print(pbacktrace, pvalue);
pyobj_str = PyObject_Str(pvalue);
py_str = py_obj_to_string(pyobj_str);
printf("%s", py_str);
myfree(py_str);
mod = PyImport_ImportModule("traceback");
list = PyObject_CallMethod(mod, "format_exception", "OOO", ptype, pvalue, pbacktrace);
if (list) {
string = PyUnicode_FromString("\n");
ret = PyUnicode_Join(string, list);
Py_DECREF(list);
Py_DECREF(string);
py_str = py_obj_to_string(ret);
printf("%s", py_str);
myfree(py_str);
Py_DECREF(ret);
}
PyErr_Clear();
}
and you will probably need this too
char *py_obj_to_string(const PyObject *py_str)
{
PyObject *py_encstr;
char *outstr = nullptr;
char *str;
py_encstr = nullptr;
str = nullptr;
if (! PyUnicode_Check((PyObject *) py_str)) {
goto err_out;
}
py_encstr = PyUnicode_AsEncodedString((PyObject *) py_str, "utf-8", nullptr);
if (! py_encstr) {
goto err_out;
}
str = PyBytes_AS_STRING(py_encstr);
if (! str) {
goto err_out;
}
outstr = strdup(str);
err_out:
if (py_encstr) {
Py_XDECREF(py_encstr);
}
return outstr;
}
actual working code if someone needs it can be found in my larger project https://github.com/goblinhack/zorbash