Develop Reference

How to use compile_commands.json with clang python bindings?

I have the following script that attempts to print out all the AST nodes in a given C++ file. This works fine when using it on a simple file with trivial includes (header file in the same directory, etc).
#!/usr/bin/env python
from argparse import ArgumentParser, FileType
from clang import cindex
def node_info(node):
return {'kind': node.kind,
'usr': node.get_usr(),
'spelling': node.spelling,
'location': node.location,
'file': node.location.file.name,
'extent.start': node.extent.start,
'extent.end': node.extent.end,
'is_definition': node.is_definition()
}
def get_nodes_in_file(node, filename, ls=None):
ls = ls if ls is not None else []
for n in node.get_children():
if n.location.file is not None and n.location.file.name == filename:
ls.append(n)
get_nodes_in_file(n, filename, ls)
return ls
def main():
arg_parser = ArgumentParser()
arg_parser.add_argument('source_file', type=FileType('r+'),
help='C++ source file to parse.')
arg_parser.add_argument('compilation_database', type=FileType('r+'),
help='The compile_commands.json to use to parse the source file.')
args = arg_parser.parse_args()
compilation_database_path = args.compilation_database.name
source_file_path = args.source_file.name
clang_args = ['-x', 'c++', '-std=c++11', '-p', compilation_database_path]
index = cindex.Index.create()
translation_unit = index.parse(source_file_path, clang_args)
file_nodes = get_nodes_in_file(translation_unit.cursor, source_file_path)
print [p.spelling for p in file_nodes]
if __name__ == '__main__':
main()
However, I get a clang.cindex.TranslationUnitLoadError: Error parsing translation unit. when I run the script and provide a valid C++ file that has a compile_commands.json file in its parent directory. This code runs and builds fine using CMake with clang, but I can't seem to figure out how to pass the argument for pointing to the compile_commands.json correctly.
I also had difficulty finding this option in the clang documentation and could not get -ast-dump to work. However, clang-check works fine by just passing the file path!

Your own accepted answer is incorrect. libclang does support compilation databases and so does cindex.py, the libclang python binding.
The main source of confusion might be that the compilation flags that libclang knows/uses are only a subset of all arguments that can be passed to the clang frontend. The compilation database is supported but does not work automatically: it must be loaded and queried manually. Something like this should work:
#!/usr/bin/env python
from argparse import ArgumentParser, FileType
from clang import cindex
compilation_database_path = args.compilation_database.name
source_file_path = args.source_file.name
index = cindex.Index.create()
# Step 1: load the compilation database
compdb = cindex.CompilationDatabase.fromDirectory(compilation_database_path)
# Step 2: query compilation flags
try:
file_args = compdb.getCompileCommands(source_file_path)
translation_unit = index.parse(source_file_path, file_args)
file_nodes = get_nodes_in_file(translation_unit.cursor, source_file_path)
print [p.spelling for p in file_nodes]
except CompilationDatabaseError:
print 'Could not load compilation flags for', source_file_path

From what I can tell Libclang does not support the compilation database but Libtooling does. To get around this I took the path to the compile_commands.json as an argument and ended up parsing it myself to find the file of interest and the relevant includes (the -I and -isystem includes).

The accepted answer seems to be deprecated, at minimum it did not work for me, I had to do this:
import clang.cindex
def main():
index = clang.cindex.Index.create()
compdb = clang.cindex.CompilationDatabase.fromDirectory(
"dir/")
source_file_path = 'path/to/file.cpp'
commands = compdb.getCompileCommands(source_file_path)
file_args = []
for command in commands:
for argument in command.arguments:
file_args.append(argument)
file_args = file_args[3:-3]
print(file_args)
translation_unit = index.parse(source_file_path, args=file_args)
comment_tokens = GetDoxygenCommentTokens(translation_unit)
if __name__ == "__main__":
main()
Basically I had to iterate over the commands and the arguments to create a string, and then eliminate some g++ specific flags.

How to override python distutils ccompiler.shared_lib_extension

I'm trying to make distutils build an extension with libtool by overriding CC, LDSHARED etc. This fails for one reason alone: I cannot, from the shell environment at least, change the extension stored in compiler.shared_lib_extension, but for libtool it must be .la for libtool to realize what it's supposed to do.
By deriving from build_ext I can override lots of things but I don't see an easy way to override what's in build_ext.run which both sets self.ccompiler and runs it.

Turns out overriding the build_ext class works:
from distutils.command.build_ext import build_ext
import re
# run the customize_compiler
class custom_build_ext(build_ext):
def build_extension(self, ext):
self.compiler.shared_lib_extension = '.la'
self.compiler.obj_extension = '.lo'
build_ext.build_extension(self,ext)
def get_ext_filename(self, ext_name):
name = build_ext.get_ext_filename(self, ext_name)
name = re.sub('\.[^.]*$', '.la', name)
return name
But installation still gives me a headache: I need to run libtool in install mode to install the .la files.
So I'm trying to override dir_util.copy_tree next.
class custom_install_lib(install_lib):
# re-assemble libtool install command
libtool_cmd = []
for env_var in ['LIBTOOL', 'AM_LIBTOOLFLAGS', 'LIBTOOLFLAGS', 'INSTALL']:
libtool_cmd += os.environ[env_var].split()
def copy_tree(self, infile, outfile,
preserve_mode=1, preserve_times=1, preserve_symlinks=0,
level=1):
"""almost identical to dir_util.copy_tree but for the special
treatment of .libs directories and .la files
"""
from distutils.file_util import copy_file
from distutils.dir_util import mkpath
verbose=1
update=0
if not self.dry_run and not os.path.isdir(infile):
raise DistutilsFileError, \
"cannot copy tree '%s': not a directory" % infile
try:
names = os.listdir(infile)
except os.error, (errno, errstr):
if self.dry_run:
names = []
else:
raise DistutilsFileError, \
"error listing files in '%s': %s" % (infile, errstr)
if not self.dry_run:
mkpath(outfile, verbose=verbose)
outputs = []
for n in names:
infile_name = os.path.join(infile, n)
outfile_name = os.path.join(outfile, n)
if n.startswith('.nfs') or n == '.libs':
# skip NFS rename files and libtool-internals
continue
if n.endswith('.la'):
print('installing libtool library', n, file=sys.stderr)
self.spawn(self.libtool_cmd + [infile_name, outfile_name])
continue
if preserve_symlinks and os.path.islink(infile_name):
link_dest = os.readlink(infile_name)
if verbose >= 1:
log.info("linking %s -> %s", outfile_name, link_dest)
if not self.dry_run:
os.symlink(link_dest, outfile_name)
outputs.append(outfile_name)
elif os.path.isdir(infile_name):
outputs.extend(
self.copy_tree(infile_name, outfile_name, preserve_mode,
preserve_times, preserve_symlinks))
else:
copy_file(infile_name, outfile_name, preserve_mode,
preserve_times, update, verbose=verbose,
dry_run=self.dry_run)
outputs.append(outfile_name)
return outputs
This is obviously much more than I had hoped for and I also needed to print
the python-computed installation directory to get the libtool -rpath
argument correct.

scon portable and cleaner way for debug and release build together

SConstruct : This file is implemented to use build library for debug and release build.
variant_dir is set to build/debug for debug build
& set to build/release for release build
import os
env = Environment()
releaseEnv = env.Clone(CCFLAGS = ['-O3'])
debugEnv = env.Clone(CCFLAGS = ['-O0', '-g'])
debugDirPath = os.path.join('build', 'debug') # build/debug
releaseDirPath = os.path.join('build', 'release') # build/release
if os.name == 'nt':
releaseEnv.Replace(CCFLAGS = ['EHsc'])
# windows specific flags
debugEnv.Replace(CCFLAGS = ['EHsc', 'Zi', 'MTd'])
SConscript(dirs = 'src', name = 'SConscript', exports = {'env' : releaseEnv}, variant_dir = releaseDirPath, duplicate = 0)
SConscript(dirs = 'src', name = 'SConscript', exports = {'env': debugEnv}, variant_dir = debugDirPath, duplicate = 0)
SConscript: (present inside source directory which contains a1.cpp and b1.cpp)
import os
Import('env')
src_list = Glob(os.path.join(Dir('#').abspath, 'src', '*.cpp'))
env.SharedLibrary(target='sum', source= src_list)
env.StaticLibrary(target='sum', source= src_list)
Directory structure is like:
root_dir -> SConstruct
-> src
-> SConscript
-> sum.cpp
-> mul.cpp
1) Running scons from root_dir generates following warning and although it's a warning message build is stop, library doesn't gets created.
scons: * Two environments with different actions were specified for the same target: /home/xyz/temp/src/mul.os
File "/home/xyz/temp/src/SConscript", line 7, in
This issue has been resolved after using src_list = Glob('*.cpp');
2) What is the proper (portable) way to create environment object for debug and release build ?
The way I have implemented is it correct ?
Kindly suggest necessary changes to avoid the warning and running build successfully.

Your problem is not related to build variants, but the fact that you have two targets with the same name (SharedLibrary and StaticLibrary both build sum).
To fix that, either just give one of them another name or add an extension to at least one of them. If you add an extension, you might want to check for OS if you want to keep your cross-platform compatibility.

Automate compilation of protobuf specs into python classes in setup.py

I have a python project that uses google protobufs as a message format for communicating over the network. Generating python files from the .proto files is straight-forward using the protoc program. How can I configure my setup.py file for the project so that it automatically calls the protoc command?

In a similar situation, I ended up with this code (setup.py, but written in a way to allow extraction into some external Python module for reuse). Note that I took the generate_proto function and several ideas from the setup.py file of the protobuf source distribution.
from __future__ import print_function
import os
import shutil
import subprocess
import sys
from distutils.command.build_py import build_py as _build_py
from distutils.command.clean import clean as _clean
from distutils.debug import DEBUG
from distutils.dist import Distribution
from distutils.spawn import find_executable
from nose.commands import nosetests as _nosetests
from setuptools import setup
PROTO_FILES = [
'goobuntu/proto/hoststatus.proto',
]
CLEANUP_SUFFIXES = [
# filepath suffixes of files to remove on "clean" subcommand
'_pb2.py',
'.pyc',
'.so',
'.o',
'dependency_links.txt',
'entry_points.txt',
'PKG-INFO',
'top_level.txt',
'SOURCES.txt',
'.coverage',
'protobuf/compiler/__init__.py',
]
CLEANUP_DIRECTORIES = [ # subdirectories to remove on "clean" subcommand
# 'build' # Note: the build subdirectory is removed if --all is set.
'html-coverage',
]
if 'PROTOC' in os.environ and os.path.exists(os.environ['PROTOC']):
protoc = os.environ['PROTOC']
else:
protoc = find_executable('protoc')
def generate_proto(source):
"""Invoke Protocol Compiler to generate python from given source .proto."""
if not os.path.exists(source):
sys.stderr.write('Can\'t find required file: %s\n' % source)
sys.exit(1)
output = source.replace('.proto', '_pb2.py')
if (not os.path.exists(output) or
(os.path.getmtime(source) > os.path.getmtime(output))):
if DEBUG:
print('Generating %s' % output)
if protoc is None:
sys.stderr.write(
'protoc not found. Is protobuf-compiler installed? \n'
'Alternatively, you can point the PROTOC environment variable at a '
'local version.')
sys.exit(1)
protoc_command = [protoc, '-I.', '--python_out=.', source]
if subprocess.call(protoc_command) != 0:
sys.exit(1)
class MyDistribution(Distribution):
# Helper class to add the ability to set a few extra arguments
# in setup():
# protofiles : Protocol buffer definitions that need compiling
# cleansuffixes : Filename suffixes (might be full names) to remove when
# "clean" is called
# cleandirectories : Directories to remove during cleanup
# Also, the class sets the clean, build_py, test and nosetests cmdclass
# options to defaults that compile protobufs, implement test as nosetests
# and enables the nosetests command as well as using our cleanup class.
def __init__(self, attrs=None):
self.protofiles = [] # default to no protobuf files
self.cleansuffixes = ['_pb2.py', '.pyc'] # default to clean generated files
self.cleandirectories = ['html-coverage'] # clean out coverage directory
cmdclass = attrs.get('cmdclass')
if not cmdclass:
cmdclass = {}
# These should actually modify attrs['cmdclass'], as we assigned the
# mutable dict to cmdclass without copying it.
if 'nosetests' not in cmdclass:
cmdclass['nosetests'] = MyNosetests
if 'test' not in cmdclass:
cmdclass['test'] = MyNosetests
if 'build_py' not in cmdclass:
cmdclass['build_py'] = MyBuildPy
if 'clean' not in cmdclass:
cmdclass['clean'] = MyClean
attrs['cmdclass'] = cmdclass
# call parent __init__ in old style class
Distribution.__init__(self, attrs)
class MyClean(_clean):
def run(self):
try:
cleandirectories = self.distribution.cleandirectories
except AttributeError:
sys.stderr.write(
'Error: cleandirectories not defined. MyDistribution not used?')
sys.exit(1)
try:
cleansuffixes = self.distribution.cleansuffixes
except AttributeError:
sys.stderr.write(
'Error: cleansuffixes not defined. MyDistribution not used?')
sys.exit(1)
# Remove build and html-coverage directories if they exist
for directory in cleandirectories:
if os.path.exists(directory):
if DEBUG:
print('Removing directory: "{}"'.format(directory))
shutil.rmtree(directory)
# Delete generated files in code tree.
for dirpath, _, filenames in os.walk('.'):
for filename in filenames:
filepath = os.path.join(dirpath, filename)
for i in cleansuffixes:
if filepath.endswith(i):
if DEBUG:
print('Removing file: "{}"'.format(filepath))
os.remove(filepath)
# _clean is an old-style class, so super() doesn't work
_clean.run(self)
class MyBuildPy(_build_py):
def run(self):
try:
protofiles = self.distribution.protofiles
except AttributeError:
sys.stderr.write(
'Error: protofiles not defined. MyDistribution not used?')
sys.exit(1)
for proto in protofiles:
generate_proto(proto)
# _build_py is an old-style class, so super() doesn't work
_build_py.run(self)
class MyNosetests(_nosetests):
def run(self):
try:
protofiles = self.distribution.protofiles
except AttributeError:
sys.stderr.write(
'Error: protofiles not defined. MyDistribution not used?')
for proto in protofiles:
generate_proto(proto)
# _nosetests is an old-style class, so super() doesn't work
_nosetests.run(self)
setup(
# MyDistribution automatically enables several extensions, including
# the compilation of protobuf files.
distclass=MyDistribution,
...
tests_require=['nose'],
protofiles=PROTO_FILES,
cleansuffixes=CLEANUP_SUFFIXES,
cleandirectories=CLEANUP_DIRECTORIES,
)

Here's the solution that I have used for setup.py. The only thing you need to keep in mind is the version of the protoc compiler is compatible with the installed protobuf version.
'''
# here you can specify the proto folder and
# output folder or input them as parameters
# to script
protoc_command = [
"python", "-m", "grpc_tools.protoc",
f"--proto_path={proto_folder}",
f"--python_out={output_folder}",
f"--grpc_python_out={output_folder}",
]
'''

Can python distutils compile CUDA code?

I have CUDA code which I want to build a dynamic library to Python using distutils. But it seems distutils doesn't recognize ".cu" file even if the "nvcc" compiler is installed. Not sure how to get it done.

Distutils is not able to compile CUDA by default, because it doesn't support using multiple compilers simultaneously. By default, it sets to compiler just based on your platform, not on the type of source code you have.
I have an example project on github that contains some monkey patches into distutils to hack in support for this. The example project is a C++ class that manages a some GPU memory and a CUDA kernel, wrapped in swig, and all compiled with just python setup.py install. The focus is on array operations, so we're also using numpy. All the kernel does for this example project is increment each element in an array by one.
The code is here: https://github.com/rmcgibbo/npcuda-example. Here's the setup.py script. The key to whole code is customize_compiler_for_nvcc().
import os
from os.path import join as pjoin
from setuptools import setup
from distutils.extension import Extension
from distutils.command.build_ext import build_ext
import subprocess
import numpy
def find_in_path(name, path):
"Find a file in a search path"
#adapted fom http://code.activestate.com/recipes/52224-find-a-file-given-a-search-path/
for dir in path.split(os.pathsep):
binpath = pjoin(dir, name)
if os.path.exists(binpath):
return os.path.abspath(binpath)
return None
def locate_cuda():
"""Locate the CUDA environment on the system
Returns a dict with keys 'home', 'nvcc', 'include', and 'lib64'
and values giving the absolute path to each directory.
Starts by looking for the CUDAHOME env variable. If not found, everything
is based on finding 'nvcc' in the PATH.
"""
# first check if the CUDAHOME env variable is in use
if 'CUDAHOME' in os.environ:
home = os.environ['CUDAHOME']
nvcc = pjoin(home, 'bin', 'nvcc')
else:
# otherwise, search the PATH for NVCC
nvcc = find_in_path('nvcc', os.environ['PATH'])
if nvcc is None:
raise EnvironmentError('The nvcc binary could not be '
'located in your $PATH. Either add it to your path, or set $CUDAHOME')
home = os.path.dirname(os.path.dirname(nvcc))
cudaconfig = {'home':home, 'nvcc':nvcc,
'include': pjoin(home, 'include'),
'lib64': pjoin(home, 'lib64')}
for k, v in cudaconfig.iteritems():
if not os.path.exists(v):
raise EnvironmentError('The CUDA %s path could not be located in %s' % (k, v))
return cudaconfig
CUDA = locate_cuda()
# Obtain the numpy include directory. This logic works across numpy versions.
try:
numpy_include = numpy.get_include()
except AttributeError:
numpy_include = numpy.get_numpy_include()
ext = Extension('_gpuadder',
sources=['src/swig_wrap.cpp', 'src/manager.cu'],
library_dirs=[CUDA['lib64']],
libraries=['cudart'],
runtime_library_dirs=[CUDA['lib64']],
# this syntax is specific to this build system
# we're only going to use certain compiler args with nvcc and not with gcc
# the implementation of this trick is in customize_compiler() below
extra_compile_args={'gcc': [],
'nvcc': ['-arch=sm_20', '--ptxas-options=-v', '-c', '--compiler-options', "'-fPIC'"]},
include_dirs = [numpy_include, CUDA['include'], 'src'])
# check for swig
if find_in_path('swig', os.environ['PATH']):
subprocess.check_call('swig -python -c++ -o src/swig_wrap.cpp src/swig.i', shell=True)
else:
raise EnvironmentError('the swig executable was not found in your PATH')
def customize_compiler_for_nvcc(self):
"""inject deep into distutils to customize how the dispatch
to gcc/nvcc works.
If you subclass UnixCCompiler, it's not trivial to get your subclass
injected in, and still have the right customizations (i.e.
distutils.sysconfig.customize_compiler) run on it. So instead of going
the OO route, I have this. Note, it's kindof like a wierd functional
subclassing going on."""
# tell the compiler it can processes .cu
self.src_extensions.append('.cu')
# save references to the default compiler_so and _comple methods
default_compiler_so = self.compiler_so
super = self._compile
# now redefine the _compile method. This gets executed for each
# object but distutils doesn't have the ability to change compilers
# based on source extension: we add it.
def _compile(obj, src, ext, cc_args, extra_postargs, pp_opts):
if os.path.splitext(src)[1] == '.cu':
# use the cuda for .cu files
self.set_executable('compiler_so', CUDA['nvcc'])
# use only a subset of the extra_postargs, which are 1-1 translated
# from the extra_compile_args in the Extension class
postargs = extra_postargs['nvcc']
else:
postargs = extra_postargs['gcc']
super(obj, src, ext, cc_args, postargs, pp_opts)
# reset the default compiler_so, which we might have changed for cuda
self.compiler_so = default_compiler_so
# inject our redefined _compile method into the class
self._compile = _compile
# run the customize_compiler
class custom_build_ext(build_ext):
def build_extensions(self):
customize_compiler_for_nvcc(self.compiler)
build_ext.build_extensions(self)
setup(name='gpuadder',
# random metadata. there's more you can supploy
author='Robert McGibbon',
version='0.1',
# this is necessary so that the swigged python file gets picked up
py_modules=['gpuadder'],
package_dir={'': 'src'},
ext_modules = [ext],
# inject our custom trigger
cmdclass={'build_ext': custom_build_ext},
# since the package has c code, the egg cannot be zipped
zip_safe=False)

As an alternative to distutils/setuptools, you could use scikit-build (along with CMakeLists.txt, pyproject.toml, and setup.cfg/setup.py):
import sys
from pathlib import Path
from skbuild import setup
from setuptools import find_packages
# https://github.com/scikit-build/scikit-build/issues/521#issuecomment-753035688
for i in (Path(__file__).resolve().parent / "_skbuild").rglob("CMakeCache.txt"):
i.write_text(re.sub("^//.*$\n^[^#].*pip-build-env.*$", "", i.read_text(), flags=re.M))
setup(cmake_args=[f"-DPython3_ROOT_DIR={sys.prefix}"],
packages=find_packages(exclude=["tests"]))