I have a large array in a file that I cannot modify but need to access in C++. I need to iterate through the array and return a value to a python script.
Problem: I am able to find the element in the array by it's name. When I try to return it's value and section elements to the python wrapper they return 0 or -1 no matter what their actual value is. Here is an example of the first element in the array from external_lists.h :
info_struct A1[] = {
{ "LIMITING", {{0x00, 0x02, 0xFF}}
.........
}
This array has thousands of similar elements. When I run the following code I get 0x or -0x1. Valuedll.cpp:
#include <iostream>
#include <string>
#include <array>
#include "external_lists.h"
extern info_struct A1[];
extern info_struct A2[];
extern int A1size;
extern int A2size;
#define DLLEXPORT extern "C" __declspec(dllexport)
DLLEXPORT int get_creation_data(const char* needed_name){
int A1_size = ( A1size/ sizeof(A1[0])) ;
int A2_size = ( A2size / sizeof(A2[0])) ;
for (int i = 0; i < A1_size; i++) {
if (A1[i].name == needed_name) {
return A1[i].style->value;
}
}
return -1;
}
The type info_struct is made from the following struct:
struct info_struct{
const char* name;
style_length style[MAX_SIZE];
options_length options[MAX_SIZE];
}
I need to get the values that are inside the style array. The style_length struct is the following:
struct style_length{
uint16_t value, section;
option_bits obits;
}
My python wrapper is the following:
import os, sys, re
from ctypes import *
import ctypes as ct
def get_creation_values(value_name):
valuell = CDLL('C:\\Documents\\creation.dll')
valuell.get_section_data.argtypes = [c_char_p]
valuell.get_section_data.restype = ct.c_int16
return hex(valuell.get_creation_data(value_name))
if __name__ == "__main__":
val = get_creation_values('LIMITING')
print(val)
Output:
(-0x1)
Thanks in advance for any help. If this is too much, thank you for reading. I will try to clarify.
I am trying to implement a Python wrapper using the Python C API over a C++ library. I need to implement conversions so I can use objects in Python and C++. I already done that in the past but I have an error I really have a hard time with.
I have a very basic test function:
PyObject* convert_to_python() {
std::cout << "Convert to PyObject" << std::endl;
long int a = 20;
PyObject* py_a = PyInt_FromLong(a);
std::cout << "Convert to PyObject ok" << std::endl;
return py_a;
}
I call this function inside a GoogleTest macro:
TEST(Wrapper, ConvertTest) {
PyObject *py_m = convert_to_python();
}
And my output is:
Convert to PyObject
Segmentation fault (core dumped)
I also ran valgrind on it:
valgrind --tool=memcheck --track-origins=yes --leak-check=full ./my_convert
But it doesn't give me much information about it:
Invalid read of size 8
==19030== at 0x4F70A7B: PyInt_FromLong (in /usr/lib/x86_64-linux-gnu/libpython2.7.so.1.0)
==19030== by 0x541E6BF: _object* pysmud_from<float>(smu::Matrix<float, 0, 0>&) (smu_type_conversions.cpp:308)
==19030== by 0x43A144: (anonymous namespace)::Wrapper_ConvertMatrix_Test::Body() (test_wrapper.cpp:12)
==19030== by 0x43A0C6: (anonymous namespace)::Wrapper_ConvertMatrix_Test::TestBody() (test_wrapper.cpp:10)
==19030== by 0x465B4D: void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (gtest.cc:2078)
==19030== by 0x460684: void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (gtest.cc:2114)
==19030== by 0x444C05: testing::Test::Run() (gtest.cc:2151)
==19030== by 0x4454C9: testing::TestInfo::Run() (gtest.cc:2326)
==19030== by 0x445BEA: testing::TestCase::Run() (gtest.cc:2444)
==19030== by 0x44CF41: testing::internal::UnitTestImpl::RunAllTests() (gtest.cc:4315)
==19030== by 0x46712C: bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) (gtest.cc:2078)
==19030== by 0x461532: bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) (gtest.cc:2114)
==19030== Address 0x0 is not stack'd, malloc'd or (recently) free'd
I think this code should work but I can't get what it's wrong with what I wrote. Did I wrongly included or linked Python files and libraries ?
EDIT: Gives no errors
#include <Python.h>
PyObject* convert_long_int(long int a) {
PyObject *ret = PyInt_FromLong(a);
return ret;
}
int main(void) {
long int a = 65454984;
PyObject *pya = convert_long_int(a);
return 0;
}
If compiling with gcc -o wraptest -I/usr/include/python2.7 wraptest.c -L/usr/lib/x86_64-linux-gnu/ -lpython2.7
What does the initialization do ?
I can confirm the segmentation fault on Ubuntu 16.04 and Python 2.7, if I omit the initialization.
Looking at Embedding Python in Another Application, there's this example
#include <Python.h>
int
main(int argc, char *argv[])
{
Py_SetProgramName(argv[0]); /* optional but recommended */
Py_Initialize();
PyRun_SimpleString("from time import time,ctime\n"
"print 'Today is',ctime(time())\n");
Py_Finalize();
return 0;
}
So when I do an equivalent minimal main
int main()
{
Py_Initialize();
PyObject *p = convert_to_python();
Py_Finalize();
return 0;
}
it works without crash.
The difference between the two examples is
long int a = 20;
and
long int a = 65454984;
I guess, it has to do with PyInt_FromLong(long ival)
The current implementation keeps an array of integer objects for all integers between -5 and 256, when you create an int in that range you actually just get back a reference to the existing object.
Maybe Python tries to access an uninitialized pointer or memory range without the initialization.
When I change the example using a = 256, it crashes. Using a = 257, it doesn't.
Looking at cpython/Objects/intobject.c:79, you can see an array of pointers
static PyIntObject *small_ints[NSMALLNEGINTS + NSMALLPOSINTS];
which is accessed right below in PyInt_FromLong(long ival)
v = small_ints[ival + NSMALLNEGINTS];
Py_INCREF(v);
But without initialization from _PyInt_Init(void)
for (ival = -NSMALLNEGINTS; ival < NSMALLPOSINTS; ival++) {
if (!free_list && (free_list = fill_free_list()) == NULL)
return 0;
/* PyObject_New is inlined */
v = free_list;
free_list = (PyIntObject *)Py_TYPE(v);
(void)PyObject_INIT(v, &PyInt_Type);
v->ob_ival = ival;
small_ints[ival + NSMALLNEGINTS] = v;
}
these pointers are all NULL, causing the crash.
I'm working on a project that need to use lib magic library to detect file mime type, I'm using 64 Bit version for windows (see: https://github.com/nscaife/file-windows) my project itself is C dll which I will call it from python. Loading the library is working fine, however when I use GetProcAddress() to access some function, it return NULL and the GetLastError() function return 126. See my code:
int DLL_EXPORT mag()
{
char *actual_file = "test.db";
const char *magic_full;
HMODULE hModule = LoadLibrary("libmagic-1.dll");
if(hModule == NULL) //No problem here
return GetLastError();
magic_t (*t0)(int) = (void *) GetProcAddress(hModule, "magic_open");
const char (*t)(magic_t, const char *) = (void *)
GetProcAddress(hModule, "magic_file");
if(t0 == NULL && t == NULL);
return GetLastError();
magic_t magic_cookie;
magic_cookie = t0(MAGIC_MIME);
magic_full = t(magic_cookie, actual_file);
return 0;
}
What is the problem Here?
I have an if clause within a for loop in which I have defined state_out beforehand with:
state_out = (PyArrayObject *) PyArray_FromDims(1,dims_new,NPY_BOOL);
And the if conditions are like this:
if (conn_ctr<sum*2){
*(state_out->data + i*state_out->strides[0]) = true;
}
else {
*(state_out->data + i*state_out->strides[0]) = false;
}
When commenting these out, state_out returns as an all-False Numpy array. There is a problem with this assignment that I fail to see. As far as I know, all within the struct PyArrayObject that are called here in this code are pointers, so after the pointer arithmetic, it should be pointing to the address I intend to write. (All if conditions in the code are built by reaching values in this manner, and I know it works, since I managed to printf input arrays' values.) Then if I want to assign a bool to one of these parts in the memory, I should assign it via *(pointer_intended) = true What am I missing?
EDIT: I have spotted that even if I don't reach those values even if I put some printf functions within:
if (conn_ctr<sum*2){
printf("True!\n");
}
else {
printf("False!\n");
}
I get a SegFault again.
Thanks a lot, an the rest of the code is here.
#include <Python.h>
#include "numpy/arrayobject.h"
#include <stdio.h>
#include <stdbool.h>
static PyObject* trace(PyObject *self, PyObject *args);
static char doc[] =
"This is the C extension for xor_masking routine. It interfaces with Python via C-Api, and calculates the"
"next state with C pointer arithmetic";
static PyMethodDef TraceMethods[] = {
{"trace", trace, METH_VARARGS, doc},
{NULL, NULL, 0, NULL}
};
PyMODINIT_FUNC
inittrace(void)
{
(void) Py_InitModule("trace", TraceMethods);
import_array();
}
static PyObject* trace(PyObject *self, PyObject *args){
PyObject *adjacency ,*mask, *state;
PyArrayObject *adjacency_arr, *mask_arr, *state_arr, *state_out;
if (!PyArg_ParseTuple(args,"OOO:trace", &adjacency, &mask, &state)) return NULL;
adjacency_arr = (PyArrayObject *)
PyArray_ContiguousFromObject(adjacency, NPY_BOOL,2,2);
if (adjacency_arr == NULL) return NULL;
mask_arr = (PyArrayObject *)
PyArray_ContiguousFromObject(mask, NPY_BOOL,2,2);
if (mask_arr == NULL) return NULL;
state_arr = (PyArrayObject *)
PyArray_ContiguousFromObject(state, NPY_BOOL,1,1);
if (state_arr == NULL) return NULL;
int dims[2], dims_new[1];
dims[0] = adjacency_arr -> dimensions[0];
dims[1] = adjacency_arr -> dimensions[1];
dims_new[0] = adjacency_arr -> dimensions[0];
if (!(dims[0]==dims[1] && mask_arr -> dimensions[0] == dims[0]
&& mask_arr -> dimensions[1] == dims[0]
&& state_arr -> dimensions[0] == dims[0]))
return NULL;
state_out = (PyArrayObject *) PyArray_FromDims(1,dims_new,NPY_BOOL);
int i,j;
for(i=0;i<dims[0];i++){
int sum = 0;
int conn_ctr = 0;
for(j=0;j<dims[1];j++){
bool adj_value = (adjacency_arr->data + i*adjacency_arr->strides[0]
+j*adjacency_arr->strides[1]);
if (*(bool *) adj_value == true){
bool mask_value = (mask_arr->data + i*mask_arr->strides[0]
+j*mask_arr->strides[1]);
bool state_value = (state_arr->data + j*state_arr->strides[0]);
if ( (*(bool *) mask_value ^ *(bool *)state_value) == true){
sum++;
}
conn_ctr++;
}
}
if (conn_ctr<sum*2){
}
else {
}
}
Py_DECREF(adjacency_arr);
Py_DECREF(mask_arr);
Py_DECREF(state_arr);
return PyArray_Return(state_out);
}
if (conn_ctr<sum*2){
*(state_out->data + i*state_out->strides[0]) = true;
}
else {
*(state_out->data + i*state_out->strides[0]) = false;
}
Here, I naively make a pointer arithmetic, state_out->data is a pointer to the beginning of data, it is defined to be a pointer of char:SciPy Doc - Python Types and C-Structures
typedef struct PyArrayObject {
PyObject_HEAD
char *data;
int nd;
npy_intp *dimensions;
npy_intp *strides;
...
} PyArrayObject;
Which a portion of I copied here. state_out->strides is a pointer to an array of length of the dimension of the array we have. This is a 1d array in this case. So when I make the pointer arithmetic (state_out->data + i*state_out->strides[0]) I certainly aim to calculate the pointer that points the ith value of the array, but I failed to give the type of the pointer, so the
I had tried :
NPY_BOOL *adj_value_ptr, *mask_value_ptr, *state_value_ptr, *state_out_ptr;
which the variables are pointing towards the values that I am interested in my for loop, and state_out_ptr is the one that I am writing to. I had thought that since I state that the
constituents of these arrays are of type NPY_BOOL, the pointers that point to the data within the array would be of type NPY_BOOL also. This fails with a SegFault when one is working with data directly manipulating the memory. This is from the fact that NPY_BOOL is an enum for an integer (as pv kindly stated in the comments.) for NumPy to use internally,.There is a C typedef npy_bool in order to use within the code for boolean values. Scipy Docs. When I introduced my pointers with the type
npy_bool *adj_value_ptr, *mask_value_ptr, *state_value_ptr, *state_out_ptr;
Segmentation fault disappeared, and I succeeded in manipulating and returning a Numpy Array.
I'm not an expert, but this solved my issue, point out if I'm wrong.
The part that has changed in the source code is:
state_out = (PyArrayObject *) PyArray_FromDims(1,dims_new,NPY_BOOL);
npy_bool *adj_value_ptr, *mask_value_ptr, *state_value_ptr, *state_out_ptr;
npy_intp i,j;
for(i=0;i<dims[0];i++){
npy_int sum = 0;
npy_int conn_ctr = 0;
for(j=0;j<dims[1];j++){
adj_value_ptr = (adjacency_arr->data + i*adjacency_arr->strides[0]
+j*adjacency_arr->strides[1]);
if (*adj_value_ptr == true){
mask_value_ptr = (mask_arr->data + i*mask_arr->strides[0]
+j*mask_arr->strides[1]);
state_value_ptr = (state_arr->data + j*state_arr->strides[0]);
if ( (*(bool *) mask_value_ptr ^ *(bool *)state_value_ptr) == true){
sum++;
}
conn_ctr++;
}
}
state_out_ptr = (state_out->data + i*state_out->strides[0]);
if (conn_ctr < sum*2){
*state_out_ptr = true;
}
else {
*state_out_ptr = false;
}
}
I'm having some trouble figuring out the proper way to walk a Python traceback using the C API. I'm writing an application that embeds the Python interpreter. I want to be able to execute arbitrary Python code, and if it raises an exception, to translate it to my own application-specific C++ exception. For now, it is sufficient to extract just the file name and line number where the Python exception was raised. This is what I have so far:
PyObject* pyresult = PyObject_CallObject(someCallablePythonObject, someArgs);
if (!pyresult)
{
PyObject* excType, *excValue, *excTraceback;
PyErr_Fetch(&excType, &excValue, &excTraceback);
PyErr_NormalizeException(&excType, &excValue, &excTraceback);
PyTracebackObject* traceback = (PyTracebackObject*)traceback;
// Advance to the last frame (python puts the most-recent call at the end)
while (traceback->tb_next != NULL)
traceback = traceback->tb_next;
// At this point I have access to the line number via traceback->tb_lineno,
// but where do I get the file name from?
// ...
}
Digging around in the Python source code, I see they access both the filename and module name of the current frame via the _frame structure, which looks like it is a privately-defined struct. My next idea was to programmatically load the Python 'traceback' module and call its functions with the C API. Is this sane? Is there a better way to access a Python traceback from C?
This is an old question but for future reference, you can get the current stack frame from the thread state object and then just walk the frames backward. A traceback object isn't necessary unless you want to preserve the state for the future.
For example:
PyThreadState *tstate = PyThreadState_GET();
if (NULL != tstate && NULL != tstate->frame) {
PyFrameObject *frame = tstate->frame;
printf("Python stack trace:\n");
while (NULL != frame) {
// int line = frame->f_lineno;
/*
frame->f_lineno will not always return the correct line number
you need to call PyCode_Addr2Line().
*/
int line = PyCode_Addr2Line(frame->f_code, frame->f_lasti);
const char *filename = PyString_AsString(frame->f_code->co_filename);
const char *funcname = PyString_AsString(frame->f_code->co_name);
printf(" %s(%d): %s\n", filename, line, funcname);
frame = frame->f_back;
}
}
I prefer calling into python from C:
err = PyErr_Occurred();
if (err != NULL) {
PyObject *ptype, *pvalue, *ptraceback;
PyObject *pystr, *module_name, *pyth_module, *pyth_func;
char *str;
PyErr_Fetch(&ptype, &pvalue, &ptraceback);
pystr = PyObject_Str(pvalue);
str = PyString_AsString(pystr);
error_description = strdup(str);
/* See if we can get a full traceback */
module_name = PyString_FromString("traceback");
pyth_module = PyImport_Import(module_name);
Py_DECREF(module_name);
if (pyth_module == NULL) {
full_backtrace = NULL;
return;
}
pyth_func = PyObject_GetAttrString(pyth_module, "format_exception");
if (pyth_func && PyCallable_Check(pyth_func)) {
PyObject *pyth_val;
pyth_val = PyObject_CallFunctionObjArgs(pyth_func, ptype, pvalue, ptraceback, NULL);
pystr = PyObject_Str(pyth_val);
str = PyString_AsString(pystr);
full_backtrace = strdup(str);
Py_DECREF(pyth_val);
}
}
I've discovered that _frame is actually defined in the frameobject.h header included with Python. Armed with this plus looking at traceback.c in the Python C implementation, we have:
#include <Python.h>
#include <frameobject.h>
PyTracebackObject* traceback = get_the_traceback();
int line = traceback->tb_lineno;
const char* filename = PyString_AsString(traceback->tb_frame->f_code->co_filename);
But this still seems really dirty to me.
One principal I've found useful in writing C extensions is to use each language where it's best suited. So if you have a task to do that would be best implemented in Python, implement in Python, and if it would be best implemented in C, do it in C. Interpreting tracebacks is best done in Python for two reasons: first, because Python has the tools to do it, and second, because it isn't speed-critical.
I would write a Python function to extract the info you need from the traceback, then call it from C.
You could even go so far as to write a Python wrapper for your callable execution. Instead of invoking someCallablePythonObject, pass it as an argument to your Python function:
def invokeSomeCallablePythonObject(obj, args):
try:
result = obj(*args)
ok = True
except:
# Do some mumbo-jumbo with the traceback, etc.
result = myTraceBackMunger(...)
ok = False
return ok, result
Then in your C code, call this Python function to do the work. The key here is to decide pragmatically which side of the C-Python split to put your code.
I used the following code to extract Python exception's error body. strExcType stores the exception type and strExcValue stores the exception body. Sample values are:
strExcType:"<class 'ImportError'>"
strExcValue:"ImportError("No module named 'nonexistingmodule'",)"
Cpp code:
if(PyErr_Occurred() != NULL) {
PyObject *pyExcType;
PyObject *pyExcValue;
PyObject *pyExcTraceback;
PyErr_Fetch(&pyExcType, &pyExcValue, &pyExcTraceback);
PyErr_NormalizeException(&pyExcType, &pyExcValue, &pyExcTraceback);
PyObject* str_exc_type = PyObject_Repr(pyExcType);
PyObject* pyStr = PyUnicode_AsEncodedString(str_exc_type, "utf-8", "Error ~");
const char *strExcType = PyBytes_AS_STRING(pyStr);
PyObject* str_exc_value = PyObject_Repr(pyExcValue);
PyObject* pyExcValueStr = PyUnicode_AsEncodedString(str_exc_value, "utf-8", "Error ~");
const char *strExcValue = PyBytes_AS_STRING(pyExcValueStr);
// When using PyErr_Restore() there is no need to use Py_XDECREF for these 3 pointers
//PyErr_Restore(pyExcType, pyExcValue, pyExcTraceback);
Py_XDECREF(pyExcType);
Py_XDECREF(pyExcValue);
Py_XDECREF(pyExcTraceback);
Py_XDECREF(str_exc_type);
Py_XDECREF(pyStr);
Py_XDECREF(str_exc_value);
Py_XDECREF(pyExcValueStr);
}
I had reason to do this recently while writing an allocation tracker for numpy. The previous answers are close but frame->f_lineno will not always return the correct line number--you need to call PyFrame_GetLineNumber(). Here's an updated code snippet:
#include "frameobject.h"
...
PyFrameObject* frame = PyEval_GetFrame();
int lineno = PyFrame_GetLineNumber(frame);
PyObject *filename = frame->f_code->co_filename;
The full thread state is also available in the PyFrameObject; if you want to walk the stack keep iterating on f_back until it's NULL. Checkout the full data structure in frameobject.h: http://svn.python.org/projects/python/trunk/Include/frameobject.h
See also: https://docs.python.org/2/c-api/reflection.html
You can access Python traceback similar to tb_printinternal function. It iterates over PyTracebackObject list. I have tried also suggestions above to iterate over frames, but it does not work for me (I see only the last stack frame).
Excerpts from CPython code:
static int
tb_displayline(PyObject *f, PyObject *filename, int lineno, PyObject *name)
{
int err;
PyObject *line;
if (filename == NULL || name == NULL)
return -1;
line = PyUnicode_FromFormat(" File \"%U\", line %d, in %U\n",
filename, lineno, name);
if (line == NULL)
return -1;
err = PyFile_WriteObject(line, f, Py_PRINT_RAW);
Py_DECREF(line);
if (err != 0)
return err;
/* ignore errors since we are not able to report them, are we? */
if (_Py_DisplaySourceLine(f, filename, lineno, 4))
PyErr_Clear();
return err;
}
static int
tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit)
{
int err = 0;
long depth = 0;
PyTracebackObject *tb1 = tb;
while (tb1 != NULL) {
depth++;
tb1 = tb1->tb_next;
}
while (tb != NULL && err == 0) {
if (depth <= limit) {
err = tb_displayline(f,
tb->tb_frame->f_code->co_filename,
tb->tb_lineno,
tb->tb_frame->f_code->co_name);
}
depth--;
tb = tb->tb_next;
if (err == 0)
err = PyErr_CheckSignals();
}
return err;
}
As of python 3.11, accessing the frame objects seems to need a different approach. Anyway, this works in 3.11, hth someone
py_err(void)
{
PyObject *err = PyErr_Occurred();
if (! err) {
return;
}
PyObject *ptype, *pvalue, *pbacktrace, *pyobj_str;
PyObject *ret, *list, *string;
PyObject *mod;
char *py_str;
PyErr_Fetch(&ptype, &pvalue, &pbacktrace);
PyErr_NormalizeException(&ptype, &pvalue, &pbacktrace);
PyErr_Display(ptype, pvalue, pbacktrace);
PyTraceBack_Print(pbacktrace, pvalue);
pyobj_str = PyObject_Str(pvalue);
py_str = py_obj_to_string(pyobj_str);
printf("%s", py_str);
myfree(py_str);
mod = PyImport_ImportModule("traceback");
list = PyObject_CallMethod(mod, "format_exception", "OOO", ptype, pvalue, pbacktrace);
if (list) {
string = PyUnicode_FromString("\n");
ret = PyUnicode_Join(string, list);
Py_DECREF(list);
Py_DECREF(string);
py_str = py_obj_to_string(ret);
printf("%s", py_str);
myfree(py_str);
Py_DECREF(ret);
}
PyErr_Clear();
}
and you will probably need this too
char *py_obj_to_string(const PyObject *py_str)
{
PyObject *py_encstr;
char *outstr = nullptr;
char *str;
py_encstr = nullptr;
str = nullptr;
if (! PyUnicode_Check((PyObject *) py_str)) {
goto err_out;
}
py_encstr = PyUnicode_AsEncodedString((PyObject *) py_str, "utf-8", nullptr);
if (! py_encstr) {
goto err_out;
}
str = PyBytes_AS_STRING(py_encstr);
if (! str) {
goto err_out;
}
outstr = strdup(str);
err_out:
if (py_encstr) {
Py_XDECREF(py_encstr);
}
return outstr;
}
actual working code if someone needs it can be found in my larger project https://github.com/goblinhack/zorbash