Develop Reference

Returning array from ctypes doesn't work with Numpy >1.14

I am trying to return an array containing 2 floats from a C++ program using Ctypes.
My C++ code looks like this:
extern "C" float* simulationSet(int cd, int ci, float cdi, int ad, int ai, float adi, int numReps){
static float retArray[2];
//DO SOME STUFF
retArray[0]=x;
retArray[1]=y;
cout<<"0="<<retArray[0]<<"\n";
cout<<"1="<<retArray[1]<<"\n";
return retArray;
}
and the Python I am using is:
import numpy as np
import sys
import ctypes
from numpy.ctypeslib import ndpointer
_ACC = ctypes.CDLL('ACC.so')
_ACC.simulationSet.argtypes = (ctypes.c_int,ctypes.c_int,ctypes.c_float,ctypes.c_int,ctypes.c_int,ctypes.c_float,ctypes.c_int)
_ACC.simulationSet.restype = ndpointer(dtype=ctypes.c_float, shape=(2,))
#_ACC.simulationSet.restype =ctypes.POINTER(ctypes.c_float*2)
ad=1
ai=3
adi=1.0
cd=1
ci=3
cdi=1.0
numReps=2
results=_ACC.simulationSet(cd,ci,cdi,ad,ai,adi,numReps)
print(results)
But, when I try to run it, I get the error:
1 3 1 1 3 1 2
0=0
1=1
Traceback (most recent call last):
File "tester.py", line 29, in <module>
results=_ACC.simulationSet(cd,ci,cdi,ad,ai,adi,numReps)
File "_ctypes/callproc.c", line 948, in GetResult
File "/usr/local/lib/python3.8/dist-packages/numpy/ctypeslib.py", line 211, in _check_retval_
return self.contents
File "/usr/local/lib/python3.8/dist-packages/numpy/ctypeslib.py", line 222, in contents
buffer = ctypes.cast(self, ctypes.POINTER(full_ctype)).contents
ValueError: NULL pointer access
and I compile with:
g++ -shared -o ACC.so CPP/*.cpp -std=c++11 -O3 -fPIC
edit: I tried it on a different machine and got this error:
File "tester.py", line 29, in <module>
results=_ACC.simulationSet(cd,ci,cdi,ad,ai,adi,numReps)
File "_ctypes/callproc.c", line 911, in GetResult
File "/home/chase/.local/lib/python3.7/site-packages/numpy/ctypeslib.py", line 183, in _check_retval_
return array(self)
ValueError: '<P' is not a valid PEP 3118 buffer format string
edit2: On the machine with the Null pointer error, its Python 3.8.5 and Numpy 1.19.0
On the version with the Pep 3118 error, its Python 3.7.0 and Numpy 1.15.0

When I use py++ to generate then boost.Python wrapper on macOS, I got a error about std::string?

The code I want to generate wrapper for is
#include <string>
#include <iostream>
class Dog
{
public:
Dog(int age, std::string name):age_(age),name_(name){}
void bark()
{
std::cout<<"Wang! Wang!"<<std::endl;
}
private:
int age_;
std::string name_;
};
And the generate python code is as follow:
from pygccxml import parser
from pyplusplus import module_builder
generator_path="/usr/local/bin/castxml"
generator_name="castxml"
compiler="clang++"
compiler_path="/usr/bin/clang++"
xml_generator_config=parser.xml_generator_configuration_t(xml_generator_path=generator_path,
xml_generator=generator_name,
compiler=compiler,
compiler_path=compiler_path)
header_collection=["Bonjour.hpp"]
builder=module_builder.module_builder_t(header_collection,xml_generator_path=generator_path,
xml_generator_config=xml_generator_config)
builder.classes().add_properties(exclude_accessors=True)
builder.build_code_creator(module_name="pylib_auto")
builder.write_module('pylib_auto.cpp')
I run with the command:
python3 pylib_generator.py
And I got the follow error:
INFO Parsing source file "Bonjour.hpp" ...
In file included from Bonjour.hpp:1:
In file included from /Library/Developer/CommandLineTools/usr/include/c++/v1/string:469:
/Library/Developer/CommandLineTools/usr/include/c++/v1/__config:235:11: fatal error: 'endian.h' file not found
# include <endian.h>
^~~~~~~~~~
1 error generated.
Traceback (most recent call last):
File "pylib_generator.py", line 17, in <module>
xml_generator_config=xml_generator_config)
File "/usr/local/lib/python3.6/site-packages/pyplusplus/module_builder/boost_python_builder.py", line 106, in __init__
, indexing_suite_version)
File "/usr/local/lib/python3.6/site-packages/pyplusplus/module_builder/boost_python_builder.py", line 149, in __parse_declarations
decls = reader.read_files( files, compilation_mode )
File "/usr/local/lib/python3.6/site-packages/pygccxml/parser/project_reader.py", line 264, in read_files
return self.__parse_file_by_file(files)
File "/usr/local/lib/python3.6/site-packages/pygccxml/parser/project_reader.py", line 292, in __parse_file_by_file
decls = reader.read_file(header)
File "/usr/local/lib/python3.6/site-packages/pygccxml/parser/source_reader.py", line 356, in read_file
return self.read_cpp_source_file(source_file)
File "/usr/local/lib/python3.6/site-packages/pygccxml/parser/source_reader.py", line 375, in read_cpp_source_file
xml_file = self.create_xml_file(ffname)
File "/usr/local/lib/python3.6/site-packages/pygccxml/parser/source_reader.py", line 324, in create_xml_file
": %s status:%s" % (gccxml_msg, exit_status))
RuntimeError: Error occurred while running CASTXML: status:1
Then I tried to do a soft link of endian.h
ln -s /usr/include/machine/endian.h /usr/local/include/endian.h
And I got the follow error:
INFO Parsing source file "Bonjour.hpp" ...
In file included from Bonjour.hpp:1:
In file included from /Library/Developer/CommandLineTools/usr/include/c++/v1/string:469:
In file included from /Library/Developer/CommandLineTools/usr/include/c++/v1/__config:235:
/usr/local/include/endian.h:37:2: error: architecture not supported
#error architecture not supported
^
In file included from Bonjour.hpp:1:
In file included from /Library/Developer/CommandLineTools/usr/include/c++/v1/string:469:
/Library/Developer/CommandLineTools/usr/include/c++/v1/__config:906:1: error: '__declspec' attributes are not enabled; use '-fdeclspec' or
'-fms-extensions' to enable support for __declspec attributes
_LIBCPP_FUNC_VIS extern "C" void __sanitizer_annotate_contiguous_container(
^
/Library/Developer/CommandLineTools/usr/include/c++/v1/__config:583:37: note: expanded from macro '_LIBCPP_FUNC_VIS'
#define _LIBCPP_FUNC_VIS _LIBCPP_DLL_VIS
^
/Library/Developer/CommandLineTools/usr/include/c++/v1/__config:576:26: note: expanded from macro '_LIBCPP_DLL_VIS'
# define _LIBCPP_DLL_VIS __declspec(dllimport)
^
/Library/Developer/CommandLineTools/usr/include/c++/v1/__config:940:4: error: "No thread API"
PS:
ProductName: Mac OS X
ProductVersion: 10.13.3
BuildVersion: 17D47
Apple LLVM version 9.0.0 (clang-900.0.39.2)
Target: x86_64-apple-darwin17.4.0
Thread model: posix
InstalledDir: /Library/Developer/CommandLineTools/usr/bin
I want to know why it got these error,seemly I can compile with clang++ .thanks anyone who can answer this!!!

I manual compile castxml on my mac, and there is no question any more. and before I just install castxml with brew. hope to be useful!

I'm not sure what makes you conclude there is a problem with std::string.
I can see a few interesting things, though:
/Library/Developer/CommandLineTools/usr/include/c++/v1/__config:940:4: error: "No thread API"
This probably indicates you need to link thread support. Compilers commonly use the -pthread flag, but you may need to also specify a linker input (instead?) on your platform. On gcc/clang it would usually be -pthread and -lpthread for the linker input
/Library/Developer/CommandLineTools/usr/include/c++/v1/__config:906:1: error: '__declspec' attributes are not enabled; use '-fdeclspec' or
'-fms-extensions' to enable support for __declspec attributes
Try using -fms-extensions or -fdeclspec

How to call a python file using Tensorflow library in C++ environment?

When I tried to call a python file using Tensorflow library in C++ environment, I got a problem like this. I have no idea how to solve it.
This is my C++ code.
#include <Python.h>
#include <iostream>
int main(int argc, wchar_t** argv)
{
const char* picpath ="/home/senius/Pictures/zys.jpg";
Py_Initialize();
//PySys_SetArgv(argc, argv);
//PyRun_SimpleString("import sys\nprint sys.argv");
if ( !Py_IsInitialized() ) {
return -1;
}
PyRun_SimpleString("import sys");
PyRun_SimpleString("sys.path.append('/home/senius/Python/Ctensor/')");
PyObject* pMod = NULL;
PyObject* pFunc = NULL;
PyObject* pParm = NULL;
PyObject* pRetVal = NULL;
int iRetVal = -999;
const char* modulName="classify"; //这个是被调用的py文件模块名字
pMod = PyImport_ImportModule(modulName);
if(!pMod)
{
std::cout<<"Import module failed!";
PyErr_Print();
return -1;
}
const char* funcName="evaluate"; //这是此py文件模块中被调用的函数名字
pFunc = PyObject_GetAttrString(pMod, funcName);
if(!pFunc)
{
std::cout<<"Import function failed!";
return -2;
}
pParm = PyTuple_New(1);
PyTuple_SetItem(pParm, 0, Py_BuildValue("s",picpath));//传入的参数，是图片的路径
pRetVal = PyEval_CallObject(pFunc, pParm);//这里开始执行py脚本
PyArg_Parse(pRetVal, "i", &iRetVal);//py脚本返回值给iRetVal
//PyErr_Print();
std::cout<<iRetVal;
return iRetVal;
}
This is my python code.
from PIL import Image
import numpy as np
import sys
import tensorflow as tf
def evaluate(pic):
if not hasattr(sys, 'argv'):
sys.argv = ['']
image = Image.open(pic)
image = image.resize([256, 256])
image_array = np.array(image)
max_index = np.argmax([1, 2, 32])
return max_index
Once I import Tensorflow in python code, there will be a error. If I don't import Tensorflow, error disappears.
Error information is as followed.
Import module failed!Traceback (most recent call last):
File "/home/senius/Python/Ctensor/classify.py", line 4, in <module>
import tensorflow as tf
File "/home/senius/anaconda3/lib/python3.6/site- packages/tensorflow/__init__.py", line 24, in <module>
from tensorflow.python import *
File "/home/senius/anaconda3/lib/python3.6/site-packages/tensorflow/python/__init__.py", line 63, in <module>
from tensorflow.python.framework.framework_lib import *
File "/home/senius/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/framework_lib.py", line 76, in <module>
from tensorflow.python.framework.ops import Graph
File "/home/senius/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 37, in <module>
from tensorflow.python.eager import context
File "/home/senius/anaconda3/lib/python3.6/site-packages/tensorflow/python/eager/context.py", line 28, in <module>
from tensorflow.python.platform import app
File "/home/senius/anaconda3/lib/python3.6/site-packages/tensorflow/python/platform/app.py", line 23, in <module>
from tensorflow.python.platform import flags
File "/home/senius/anaconda3/lib/python3.6/site-packages/tensorflow/python/platform/flags.py", line 26, in <module>
_global_parser = _argparse.ArgumentParser()
File "/home/senius/anaconda3/lib/python3.6/argparse.py", line 1622, in __init__
prog = _os.path.basename(_sys.argv[0])
AttributeError: module 'sys' has no attribute 'argv'

You import the tensorflow module - and during that import it attempts to use sys.argv (as you can see from your stack trade) and then sometime later in some function, you have
if not hasattr(sys, 'argv'):
sys.argv = ['']`
which is the workaround to this error - but it can never be called because the import failed.
The fix is to move this to before you import the tensorflow module

How to use const in Cython

I have read in this link: https://github.com/cython/cython/wiki/FAQ#id35 that my Cython 0.20.1 should be able to support const.
However my following code does not compile:
cdef const double a = 2.5
print(a)
During compilation it says
test.pyx:5:5: Assignment to const 'a'
Traceback (most recent call last):
File "setup1.py", line 11, in <module>
ext_modules = cythonize(extensions)
File "X:\WinPython3\python-3.3.5.amd64\lib\site-packages\Cython\Build\Dependencies.py", line 785, in cythonize
cythonize_one(*args[1:])
File "X:\WinPython3\python-3.3.5.amd64\lib\site-packages\Cython\Build\Dependencies.py", line 902, in cythonize_one
raise CompileError(None, pyx_file)
Cython.Compiler.Errors.CompileError: test.pyx

const works for function arguments, but not for this construct because of how Cython compiles it to C.
cdef double a = 2.5
becomes
double __pyx_v_a;
/* lots of other declarations */
__pyx_v_a = 2.5;
and this wouldn't work with a const variable. In effect, you cannot use const in Cython in all the places were you can use it in C.

pyopencl.RuntimeError: clBuildProgram failed: invalid build options

I am new to OpenCL and have some problems with setting up a OpenCL program. To illustrate my problem please look at the code (taken from https://github.com/benshope/PyOpenCL-Tutorial):
# Use OpenCL To Add Two Random Arrays (This Way Hides Details)
import pyopencl as cl # Import the OpenCL GPU computing API
import pyopencl.array as pycl_array # Import PyOpenCL Array (a Numpy array plus an OpenCL buffer object)
import numpy as np # Import Numpy number tools
context = cl.create_some_context() # Initialize the Context
queue = cl.CommandQueue(context) # Instantiate a Queue
a = pycl_array.to_device(queue, np.random.rand(50000).astype(np.float32))
b = pycl_array.to_device(queue, np.random.rand(50000).astype(np.float32))
# Create two random pyopencl arrays
c = pycl_array.empty_like(a) # Create an empty pyopencl destination array
program = cl.Program(context, """
__kernel void sum(__global const float *a, __global const float *b, __global float *c)
{
int i = get_global_id(0);
c[i] = a[i] + b[i];
}""").build() # Create the OpenCL program
program.sum(queue, a.shape, None, a.data, b.data, c.data) # Enqueue the program for execution and store the result in c
print("a: {}".format(a))
print("b: {}".format(b))
print("c: {}".format(c))
# Print all three arrays, to show sum() worked
If I execute the script I get the following error:
"C:\Program Files\WinPython-64bit-2.7.6.3\python-2.7.6.amd64\python.exe" D:/python/openCL/020_array_sum.py
Traceback (most recent call last):
File "D:/python/openCL/020_array_sum.py", line 20, in <module>
}""").build() # Create the OpenCL program
File "C:\Program Files\WinPython-64bit-2.7.6.3\python-2.7.6.amd64\lib\site-packages\pyopencl\__init__.py", line 166, in build
options=options, source=self._source)
File "C:\Program Files\WinPython-64bit-2.7.6.3\python-2.7.6.amd64\lib\site-packages\pyopencl\__init__.py", line 206, in _build_and_catch_errors
raise err
pyopencl.RuntimeError: clBuildProgram failed: invalid build options -
(options: -I c:\program files\winpython-64bit-2.7.6.3\python-2.7.6.amd64\lib\site-packages\pyopencl\cl)
(source saved as c:\appdata\local\temp\tmp0bj_ij.cl)
Process finished with exit code 1
As far as I understood it, this is caused by the build() function, but I do not understand why. In one forum they suggested to define the kernel with only one " instead of """. This also did not help.
I use WinPython-64bit-2.7.6.3 and pycharm-community-3.1.1. For the openCL i have installed: AMD-APP-SDK-v2.9-Windows-641, Mako-0.9.1.win-amd64-py2.7, pytools-2014.1.2.win-amd64-py2.7 and pyopencl-2013.2.win-amd64-py2.7.
My graphics card is a Radeon HD 7850 and I have a AMD PhenomII processor.
P.S.: When I compile in Spyder, the error message reads:
>>> runfile('D:/python/openCL/020_array_sum.py', wdir=r'D:/python/openCL')
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "C:\Program Files\WinPython-64bit-2.7.6.3\python-2.7.6.amd64\lib\site-packages\spyderlib\widgets\externalshell\sitecustomize.py", line 540, in runfile
execfile(filename, namespace)
File "D:/python/openCL/020_array_sum.py", line 20, in <module>
}""").build() # Create the OpenCL program
File "C:\Program Files\WinPython-64bit-2.7.6.3\python-2.7.6.amd64\lib\site-packages\pyopencl\__init__.py", line 166, in build
options=options, source=self._source)
File "C:\Program Files\WinPython-64bit-2.7.6.3\python-2.7.6.amd64\lib\site-packages\pyopencl\__init__.py", line 206, in _build_and_catch_errors
raise err
pyopencl.RuntimeError: clBuildProgram failed: invalid build options -
(options: -I c:\program files\winpython-64bit-2.7.6.3\python-2.7.6.amd64\lib\site-packages\pyopencl\cl)
(source saved as c:\users\andreas\appdata\local\temp\tmpzrgacv.cl)
Edit: I have now also tested it on another PC: same error. It also has an Nvidia graphics card. What both have in commen, is that they are only specified with OpenCL 1.1. Could it be, that I need OpenCL 1.2?

I think I found the problem. I changed the installtion directory of WinPython such that the path no longer includes spaces, now C:\WinPython-64bit-2.7.6.3. Then it worked. Thanks again for all your suggestions and your time.

This worked for me, I disabled caching of built kernels. My solution is useful if you are using pyopencl
import os
os.environ['PYOPENCL_NO_CACHE'] = '1'