I am trying to use sphinx for the first time. I'm using sphinx3 because I need s3_aligner so I followed many tutorials but none of them got me a result I always get this error :
**configure: error: in `/Users/Bassem/downloads/cmusphinx/sphinxbase':
configure: error:
Could not link test program to Python. Maybe the main Python library has been
installed in some non-standard library path. If so, pass it to configure,
via the LDFLAGS environment variable.
Example: ./configure LDFLAGS="-L/usr/non-standard-path/python/lib"
============================================================================
ERROR!
You probably have to install the development version of the Python package
for your distribution. The exact name of this package varies among them.
===========================================================================**
I tried so many solution but none of them helped me out all the solutions give the same error .There is a solution for ubuntu which is python-dev for easy linking . I need something equivalent to it on mac or anything to solve me the problem
Here is my config.log:
## ---------------- ##
## Cache variables. ##
## ---------------- ##
ac_cv_build=x86_64-apple-darwin15.3.0
ac_cv_c_bigendian=no
ac_cv_c_compiler_gnu=yes
ac_cv_c_const=yes
ac_cv_env_CC_set=
ac_cv_env_CC_value=
ac_cv_env_CFLAGS_set=
ac_cv_env_CFLAGS_value=
ac_cv_env_CPPFLAGS_set=
ac_cv_env_CPPFLAGS_value=
ac_cv_env_CPP_set=
ac_cv_env_CPP_value=
ac_cv_env_LDFLAGS_set=
ac_cv_env_LDFLAGS_value=
ac_cv_env_LIBS_set=
ac_cv_env_LIBS_value=
ac_cv_env_LT_SYS_LIBRARY_PATH_set=
ac_cv_env_LT_SYS_LIBRARY_PATH_value=
ac_cv_env_build_alias_set=
ac_cv_env_build_alias_value=
ac_cv_env_host_alias_set=
ac_cv_env_host_alias_value=
ac_cv_env_target_alias_set=
ac_cv_env_target_alias_value=
ac_cv_file__Users_Bassem_downloads_cmusphinx_sphinx3____sphinxbase_include_sphinxbase_prim_type_h=yes
ac_cv_file__Users_Bassem_downloads_cmusphinx_sphinx3____sphinxbase_src_libsphinxbase_libsphinxbase_la=no
ac_cv_func_bcopy=yes
ac_cv_func_memmove=yes
ac_cv_header_dlfcn_h=yes
ac_cv_header_inttypes_h=yes
ac_cv_header_memory_h=yes
ac_cv_header_sphinxbase_sphinx_config_h=no
ac_cv_header_stdc=yes
ac_cv_header_stdint_h=yes
ac_cv_header_stdlib_h=yes
ac_cv_header_string_h=yes
ac_cv_header_strings_h=yes
ac_cv_header_sys_stat_h=yes
ac_cv_header_sys_types_h=yes
ac_cv_header_unistd_h=yes
ac_cv_host=x86_64-apple-darwin15.3.0
ac_cv_objext=o
ac_cv_path_EGREP='/usr/bin/grep -E'
ac_cv_path_FGREP='/usr/bin/grep -F'
ac_cv_path_GREP=/usr/bin/grep
ac_cv_path_SED=/usr/bin/sed
ac_cv_path_install='/usr/bin/install -c'
ac_cv_path_lt_DD=/bin/dd
ac_cv_prog_AWK=awk
ac_cv_prog_CPP='gcc -E'
ac_cv_prog_ac_ct_AR=ar
ac_cv_prog_ac_ct_CC=gcc
ac_cv_prog_ac_ct_DSYMUTIL=dsymutil
ac_cv_prog_ac_ct_LIPO=lipo
ac_cv_prog_ac_ct_NMEDIT=nmedit
ac_cv_prog_ac_ct_OTOOL=otool
ac_cv_prog_ac_ct_RANLIB=ranlib
ac_cv_prog_ac_ct_STRIP=strip
ac_cv_prog_cc_c89=
ac_cv_prog_cc_g=yes
ac_cv_prog_make_make_set=yes
ac_cv_search_strerror='none required'
ac_cv_type_signal=void
ac_cv_type_size_t=yes
am_cv_CC_dependencies_compiler_type=gcc3
am_cv_make_support_nested_variables=yes
am_cv_prog_cc_c_o=yes
am_cv_prog_cc_stdc=
cc_cv_attribute_visibility=yes
lt_cv_apple_cc_single_mod=yes
lt_cv_ar_at_file=no
lt_cv_deplibs_check_method=pass_all
lt_cv_file_magic_cmd='$MAGIC_CMD'
lt_cv_file_magic_test_file=
lt_cv_ld_exported_symbols_list=yes
lt_cv_ld_force_load=yes
lt_cv_ld_reload_flag=-r
lt_cv_nm_interface='BSD nm'
lt_cv_objdir=.libs
lt_cv_path_LD=/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/ld
lt_cv_path_NM=/usr/bin/nm
lt_cv_path_mainfest_tool=no
lt_cv_prog_compiler_c_o=yes
lt_cv_prog_compiler_pic='-fno-common -DPIC'
lt_cv_prog_compiler_pic_works=yes
lt_cv_prog_compiler_rtti_exceptions=yes
lt_cv_prog_compiler_static_works=no
lt_cv_prog_gnu_ld=no
lt_cv_sharedlib_from_linklib_cmd='printf %s\n'
lt_cv_sys_global_symbol_pipe='sed -n -e '\''s/^.*[ ]\([BCDEGRST][BCDEGRST]*\)[ ][ ]*_\([_A-Za-z][_A-Za-z0-9]*\)$/\1 _\2 \2/p'\'' | sed '\''/ __gnu_lto/d'\'''
lt_cv_sys_global_symbol_to_c_name_address='sed -n -e '\''s/^: \(.*\) .*$/ {"\1", (void *) 0},/p'\'' -e '\''s/^[BCDEGRST][BCDEGRST]* .* \(.*\)$/ {"\1", (void *) \&\1},/p'\'''
lt_cv_sys_global_symbol_to_c_name_address_lib_prefix='sed -n -e '\''s/^: \(.*\) .*$/ {"\1", (void *) 0},/p'\'' -e '\''s/^[BCDEGRST][BCDEGRST]* .* \(lib.*\)$/ {"\1", (void *) \&\1},/p'\'' -e '\''s/^[BCDEGRST][BCDEGRST]* .* \(.*\)$/ {"lib\1", (void *) \&\1},/p'\'''
lt_cv_sys_global_symbol_to_cdecl='sed -n -e '\''s/^T .* \(.*\)$/extern int \1();/p'\'' -e '\''s/^[BCDEGRST][BCDEGRST]* .* \(.*\)$/extern char \1;/p'\'''
lt_cv_sys_global_symbol_to_import=
lt_cv_sys_max_cmd_len=196608
lt_cv_to_host_file_cmd=func_convert_file_noop
lt_cv_to_tool_file_cmd=func_convert_file_noop
lt_cv_truncate_bin='/bin/dd bs=4096 count=1'
## ----------------- ##
## Output variables. ##
## ----------------- ##
ACLOCAL='${SHELL} /Users/Bassem/downloads/cmusphinx/sphinx3/missing aclocal-1.15'
AMDEPBACKSLASH='\'
AMDEP_FALSE='#'
AMDEP_TRUE=''
AMTAR='$${TAR-tar}'
AM_BACKSLASH='\'
AM_DEFAULT_V='$(AM_DEFAULT_VERBOSITY)'
AM_DEFAULT_VERBOSITY='1'
AM_V='$(V)'
AR='ar'
AUTOCONF='${SHELL} /Users/Bassem/downloads/cmusphinx/sphinx3/missing autoconf'
AUTOHEADER='${SHELL} /Users/Bassem/downloads/cmusphinx/sphinx3/missing autoheader'
AUTOMAKE='${SHELL} /Users/Bassem/downloads/cmusphinx/sphinx3/missing automake-1.15'
AWK='awk'
CC='gcc'
CCDEPMODE='depmode=gcc3'
CFLAGS='-g -O2 -Wall -fvisibility=hidden'
CPP='gcc -E'
CPPFLAGS='-I/Users/Bassem/downloads/cmusphinx/sphinx3/../sphinxbase/include '
CSH=''
CYGPATH_W='echo'
DEFS=''
DEPDIR='.deps'
DLLTOOL='false'
DSYMUTIL='dsymutil'
DUMPBIN=''
ECHO_C='\c'
ECHO_N=''
ECHO_T=''
EGREP='/usr/bin/grep -E'
EXEEXT=''
FGREP='/usr/bin/grep -F'
GREP='/usr/bin/grep'
INSTALL_DATA='${INSTALL} -m 644'
INSTALL_PROGRAM='${INSTALL}'
INSTALL_SCRIPT='${INSTALL}'
INSTALL_STRIP_PROGRAM='$(install_sh) -c -s'
LD='/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/ld'
LDFLAGS=''
LIBOBJS=''
LIBS=''
LIBTOOL='$(SHELL) $(top_builddir)/libtool'
LIPO='lipo'
LN_S='ln -s'
LTLIBOBJS=''
LT_SYS_LIBRARY_PATH=''
MAKEINFO='${SHELL} /Users/Bassem/downloads/cmusphinx/sphinx3/missing makeinfo'
MANIFEST_TOOL=':'
MKDIR_P='./install-sh -c -d'
NM='/usr/bin/nm'
NMEDIT='nmedit'
OBJDUMP='false'
OBJEXT='o'
OTOOL64=':'
OTOOL='otool'
PACKAGE='sphinx3'
PACKAGE_BUGREPORT=''
PACKAGE_NAME='sphinx3'
PACKAGE_STRING='sphinx3 0.7'
PACKAGE_TARNAME='sphinx3'
PACKAGE_URL=''
PACKAGE_VERSION='0.7'
PATH_SEPARATOR=':'
PERL=''
RANLIB='ranlib'
SED='/usr/bin/sed'
SET_MAKE=''
SHELL='/bin/sh'
STRIP='strip'
VERSION='0.7'
ac_ct_AR='ar'
ac_ct_CC='gcc'
ac_ct_DUMPBIN=''
ad_backend=''
ad_files=''
ad_libs=''
am__EXEEXT_FALSE=''
am__EXEEXT_TRUE=''
am__fastdepCC_FALSE='#'
am__fastdepCC_TRUE=''
am__include='include'
am__isrc=''
am__leading_dot='.'
am__nodep='_no'
am__quote=''
am__tar='$${TAR-tar} chof - "$$tardir"'
am__untar='$${TAR-tar} xf -'
bindir='${exec_prefix}/bin'
build='x86_64-apple-darwin15.3.0'
build_alias=''
build_cpu='x86_64'
build_os='darwin15.3.0'
build_vendor='apple'
datadir='${datarootdir}'
datarootdir='${prefix}/share'
docdata='./.DS_Store ./BN_AM_HUB96-97.doc ./BN_AM_HUB96-97.htm ./cmdhelp.txt ./doxygen.cfg ./doxygen.main ./images/rarrow.gif ./index.html ./models.html ./rarrow.gif ./s3/falign.fig ./s3/falign.gif ./s3/feat.fig ./s3/feat.gif ./s3/hypseg.txt ./s3-2.htm ./s3-2.pdf ./s3-2.ppt ./s3-2_files/buttons.gif ./s3-2_files/error.htm ./s3-2_files/filelist.xml ./s3-2_files/frame.htm ./s3-2_files/fullscreen.htm ./s3-2_files/master01.htm ./s3-2_files/master02.htm ./s3-2_files/master03.htm ./s3-2_files/master03.xml ./s3-2_files/master03_image002.gif ./s3-2_files/master03_stylesheet.css ./s3-2_files/outline.htm ./s3-2_files/pres.xml ./s3-2_files/preview.wmf ./s3-2_files/script.js ./s3-2_files/slide0001.htm ./s3-2_files/slide0001_image001.gif ./s3-2_files/slide0004.htm ./s3-2_files/slide0005.htm ./s3-2_files/slide0006.htm ./s3-2_files/slide0007.htm ./s3-2_files/slide0008.htm ./s3-2_files/slide0009.htm ./s3-2_files/slide0010.htm ./s3-2_files/slide0011.htm ./s3-2_files/slide0012.htm ./s3-2_files/slide0012_image003.gif ./s3-2_files/slide0015.htm ./s3-2_files/slide0015_image004.gif ./s3-2_files/slide0015_image005.gif ./s3-2_files/slide0015_image006.gif ./s3-2_files/slide0015_image007.gif ./s3-2_files/slide0015_image008.gif ./s3-2_files/slide0016.htm ./s3-2_files/slide0016.xml ./s3-2_files/slide0016_image009.gif ./s3-2_files/slide0016_image010.gif ./s3-2_files/slide0016_image011.gif ./s3-2_files/slide0016_image012.gif ./s3-2_files/slide0016_image013.gif ./s3-2_files/slide0016_image014.gif ./s3-2_files/slide0016_image015.gif ./s3-2_files/slide0016_image016.gif ./s3-2_files/slide0016_image017.gif ./s3-2_files/slide0016_image018.gif ./s3-2_files/slide0016_image019.gif ./s3-2_files/slide0016_image020.gif ./s3-2_files/slide0017.htm ./s3-2_files/slide0017.xml ./s3-2_files/slide0017_image021.gif ./s3-2_files/slide0017_image022.gif ./s3-2_files/slide0017_image023.gif ./s3-2_files/slide0017_image024.gif ./s3-2_files/slide0017_image025.gif ./s3-2_files/slide0017_image026.gif ./s3-2_files/slide0017_image027.gif ./s3-2_files/slide0017_image028.gif ./s3-2_files/slide0017_image029.gif ./s3-2_files/slide0017_image030.gif ./s3-2_files/slide0017_image031.gif ./s3-2_files/slide0018.htm ./s3-2_files/slide0018.xml ./s3-2_files/slide0018_image032.gif ./s3-2_files/slide0019.htm ./s3-2_files/slide0020.htm ./s3-2_files/slide0020_image033.gif ./s3-2_files/slide0021.htm ./s3-2_files/slide0021_image034.gif ./s3-2_files/slide0022.htm ./s3-2_files/slide0022_image035.gif ./s3-2_files/slide0023.htm ./s3-2_files/slide0023_image036.gif ./s3-2_files/slide0024.htm ./s3-2_files/slide0024_image037.gif ./s3-2_files/slide0026.htm ./s3-2_files/slide0026_image038.gif ./s3-2_files/slide0027.htm ./s3-2_files/slide0028.htm ./s3-2_files/slide0028_image039.gif ./s3-2_files/slide0029.htm ./s3-2_files/slide0029.xml ./s3-2_files/slide0029_image040.gif ./s3-2_files/slide0030.htm ./s3-2_files/slide0030.xml ./s3-2_files/slide0030_image041.gif ./s3-2_files/slide0031.htm ./s3-2_files/slide0031_image042.gif ./s3-2_files/slide0032.htm ./s3-2_files/slide0032_image043.gif ./s3-2_files/slide0033.htm ./s3-2_files/slide0034.htm ./s3-2_files/slide0034_image044.gif ./s3-2_files/slide0035.htm ./s3-2_files/slide0035_image045.gif ./s3-2_files/slide0036.htm ./s3-2_files/slide0036_image046.gif ./s3-2_files/slide0037.htm ./s3-2_files/slide0037_image047.gif ./s3-2_files/slide0038.htm ./s3-2_files/slide0038_image048.gif ./s3-2_files/slide0039.htm ./s3-2_files/slide0040.htm ./s3-2_files/slide0041.htm ./s3-4.pdf ./s3-4.ppt ./s3-5.ppt ./s3-6.ppt ./s3_codewalk.html ./s3_description.html ./s3_fe_spec.pdf ./s3_overview.html ./sphinx3.3-6.ppt ./sphinx3.4.code.tracing ./sphinx3.5_refactoring.note ./sphinxman_FAQ.html ./sphinxman_manual.html ./sphinxman_misc.html'
docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
dvidir='${docdir}'
epdata='./chan3-dither.cepview ./chan3-logspec.cepview ./chan3.cepview ./chan3.logspec ./chan3.mfc ./chan3.raw ./ep.mdef ./ep.result ./means ./mixture_weights ./variances'
exec_prefix='NONE'
hmmdata='./8gau.6000sen.quant ./feat.params ./mdef ./means ./mixture_weights ./mllr_matrices ./test-align-mllr.out ./test-conf.confhypseg ./test.align.mllr.out ./test.align.out ./test.align.phseg ./test.align.wdseg ./test.allphone.allp ./test.allphone.match ./test.allphone.matchseg ./test.allphone.mllr.allp ./test.allphone.mllr.match ./test.allphone.mllr.matchseg ./test.allphone.phone_tg.allp ./test.allphone.phone_tg.match ./test.allphone.phone_tg.matchseg ./test.allphone.phone_tg.mllr.allp ./test.allphone.phone_tg.mllr.match ./test.allphone.phone_tg.mllr.matchseg ./test.dp.hyp ./test.dp.ref ./test.dp.simple.log ./test.mode1369.dump ./test.subvq ./transition_matrices ./variances'
host='x86_64-apple-darwin15.3.0'
host_alias=''
host_cpu='x86_64'
host_os='darwin15.3.0'
host_vendor='apple'
htmldir='${docdir}'
includedir='${prefix}/include'
infodir='${datarootdir}/info'
install_sh='${SHELL} /Users/Bassem/downloads/cmusphinx/sphinx3/install-sh'
libdir='${exec_prefix}/lib'
libexecdir='${exec_prefix}/libexec'
lmdata='./README ./align.correct ./an4.cls.probdef ./an4.ctl ./an4.ctl.platform_independent ./an4.ctl_lm ./an4.dict ./an4.phone.dict ./an4.phonelist ./an4.tg.phone.arpa ./an4.tg.phone.arpa.DMP ./an4.tg.phone.arpa.FST ./an4.tg.phone.arpa.FST.SYM ./an4.tg.phone.arpa.lm_convert ./an4.ug.cls.lm ./an4.ug.cls.lm.DMP ./an4.ug.cls.lmctl ./an4.ug.fsg ./an4.ug.lm ./an4.ug.lm.DMP ./an4.ug.lm.FST ./an4.ug.lm.FST.SYM ./an4.ug.lm.lm_convert ./args.an4 ./args.an4.test ./args.an4.test.cls ./args.an4.test.fsg ./args.an4.test.mllr ./args.an4.test.win32 ./filler.dict ./pittsburgh.bigendian.mfc ./pittsburgh.bigendian.raw ./pittsburgh.lat.gz ./pittsburgh.littleendian.abcd ./pittsburgh.littleendian.mfc ./pittsburgh.littleendian.raw ./pittsburgh.littleendian.slf ./pittsburgh.nbest'
localedir='${datarootdir}/locale'
localstatedir='${prefix}/var'
mandir='${datarootdir}/man'
mkdir_p='$(MKDIR_P)'
oldincludedir='/usr/include'
pdfdir='${docdir}'
prefix='NONE'
program_transform_name='s,x,x,'
psdir='${docdir}'
sbindir='${exec_prefix}/sbin'
sharedstatedir='${prefix}/com'
sphinxbase='/Users/Bassem/downloads/cmusphinx/sphinx3/../sphinxbase'
sysconfdir='${prefix}/etc'
target_alias=''
tidigitsdata='./cepstra/man/man.ah.111a.mfc ./cepstra/man/man.ah.1b.mfc ./cepstra/man/man.ah.2934za.mfc ./cepstra/man/man.ah.35oa.mfc ./cepstra/man/man.ah.3oa.mfc ./cepstra/man/man.ah.4625a.mfc ./cepstra/man/man.ah.588zza.mfc ./cepstra/man/man.ah.63a.mfc ./cepstra/man/man.ah.6o838a.mfc ./cepstra/man/man.ah.75913a.mfc ./cepstra/man/man.ah.844o1a.mfc ./cepstra/man/man.ah.8b.mfc ./cepstra/man/man.ah.9b.mfc ./cepstra/man/man.ah.o789a.mfc ./cepstra/man/man.ah.z4548a.mfc ./cepstra/man/man.ah.zb.mfc ./cepstra/woman/woman.ak.1b.mfc ./cepstra/woman/woman.ak.276317oa.mfc ./cepstra/woman/woman.ak.334a.mfc ./cepstra/woman/woman.ak.3z3z9a.mfc ./cepstra/woman/woman.ak.48z66zza.mfc ./cepstra/woman/woman.ak.532a.mfc ./cepstra/woman/woman.ak.5z874a.mfc ./cepstra/woman/woman.ak.6728za.mfc ./cepstra/woman/woman.ak.75a.mfc ./cepstra/woman/woman.ak.84983a.mfc ./cepstra/woman/woman.ak.8a.mfc ./cepstra/woman/woman.ak.99731a.mfc ./cepstra/woman/woman.ak.o69a.mfc ./cepstra/woman/woman.ak.ooa.mfc ./cepstra/woman/woman.ak.za.mfc ./dictionary ./fillerdict ./test.2.digits.fsg ./test.digits.fsg ./test.iso.digits.fsg ./tidigits.length.1.regression ./tidigits.length.1.result ./tidigits.length.2.regression ./tidigits.length.2.result ./tidigits.length.arb.regression ./tidigits.length.arb.result ./wd_dependent_phone.cd_continuous_8gau/mdef ./wd_dependent_phone.cd_continuous_8gau/means ./wd_dependent_phone.cd_continuous_8gau/mixture_weights ./wd_dependent_phone.cd_continuous_8gau/transition_matrices ./wd_dependent_phone.cd_continuous_8gau/variances'
## ----------- ##
## confdefs.h. ##
## ----------- ##
/* confdefs.h */
#define PACKAGE_NAME "sphinx3"
#define PACKAGE_TARNAME "sphinx3"
#define PACKAGE_VERSION "0.7"
#define PACKAGE_STRING "sphinx3 0.7"
#define PACKAGE_BUGREPORT ""
#define PACKAGE_URL ""
#define STDC_HEADERS 1
#define HAVE_SYS_TYPES_H 1
#define HAVE_SYS_STAT_H 1
#define HAVE_STDLIB_H 1
#define HAVE_STRING_H 1
#define HAVE_MEMORY_H 1
#define HAVE_STRINGS_H 1
#define HAVE_INTTYPES_H 1
#define HAVE_STDINT_H 1
#define HAVE_UNISTD_H 1
#define STDC_HEADERS 1
#define HAVE_MEMMOVE 1
#define HAVE_BCOPY 1
#define RETSIGTYPE void
#define HAVE_DLFCN_H 1
#define LT_OBJDIR ".libs/"
#define HAVE_ATTRIBUTE_VISIBILITY 1
configure: exit 1
Related
New at this so please bare with me. Following these instructions trying update to the latest version of Python
https://www.vultr.com/docs/update-python3-on-centos/
I get all the way to step 2.5 and get the following:
[user1#localhost Python-3.9.6]$ ./configure --enable-optimizations
checking build system type... x86_64-pc-linux-gnu
checking host system type... x86_64-pc-linux-gnu
checking for python3.9... no
checking for python3... python3
checking for --enable-universalsdk... no
checking for --with-universal-archs... no
checking MACHDEP... "linux"
checking for gcc... gcc
checking whether the C compiler works... no
configure: error: in `/home/user1/Python-3.9.6':
configure: error: C compiler cannot create executables
See `config.log' for more details
My config.log file
## --------- ##
## Platform. ##
## --------- ##
hostname = localhost.localdomain
uname -m = x86_64
uname -r = 3.10.0-1160.76.1.el7.x86_64
uname -s = Linux
uname -v = #1 SMP Wed Aug 10 16:21:17 UTC 2022
/usr/bin/uname -p = x86_64
/bin/uname -X = unknown
/bin/arch = x86_64
/usr/bin/arch -k = unknown
/usr/convex/getsysinfo = unknown
/usr/bin/hostinfo = unknown
/bin/machine = unknown
/usr/bin/oslevel = unknown
/bin/universe = unknown
PATH: /opt/rh/devtoolset-8/root/usr/bin
PATH: /usr/local/bin
PATH: /usr/bin
PATH: /usr/local/sbin
PATH: /usr/sbin
PATH: /home/user1/.local/bin
PATH: /home/user1/bin
## ----------- ##
## Core tests. ##
## ----------- ##
configure:2848: checking build system type
configure:2862: result: x86_64-pc-linux-gnu
configure:2882: checking host system type
configure:2895: result: x86_64-pc-linux-gnu
configure:2925: checking for python3.9
configure:2955: result: no
configure:2925: checking for python3
configure:2941: found /usr/bin/python3
configure:2952: result: python3
configure:3046: checking for --enable-universalsdk
configure:3093: result: no
configure:3117: checking for --with-universal-archs
configure:3132: result: no
configure:3288: checking MACHDEP
configure:3339: result: "linux"
configure:3633: checking for gcc
configure:3649: found /opt/rh/devtoolset-8/root/usr/bin/gcc
configure:3660: result: gcc
configure:3889: checking for C compiler version
configure:3898: gcc --version >&5
gcc (GCC) 8.3.1 20190311 (Red Hat 8.3.1-3)
Copyright (C) 2018 Free Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
configure:3909: $? = 0
configure:3898: gcc -v >&5
Using built-in specs.
COLLECT_GCC=gcc
COLLECT_LTO_WRAPPER=/opt/rh/devtoolset-8/root/usr/libexec/gcc/x86_64-redhat-linux/8/lto-wrapper
Target: x86_64-redhat-linux
Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,fortran,lto --prefix=/opt/rh/devtoolset-8/root/usr --mandir=/opt/rh/devtoolset-8/root/usr/share/man --infodir=/opt/rh/devtoolset-8/root/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --with-default-libstdcxx-abi=gcc4-compatible --enable-plugin --enable-initfini-array --with-isl=/builddir/build/BUILD/gcc-8.3.1-20190311/obj-x86_64-redhat-linux/isl-install --disable-libmpx --enable-gnu-indirect-function --with-tune=generic --with-arch_32=x86-64 --build=x86_64-redhat-linux
Thread model: posix
gcc version 8.3.1 20190311 (Red Hat 8.3.1-3) (GCC)
configure:3909: $? = 0
configure:3898: gcc -V >&5
gcc: error: unrecognized command line option '-V'
gcc: fatal error: no input files
compilation terminated.
configure:3909: $? = 1
configure:3898: gcc -qversion >&5
gcc: error: unrecognized command line option '-qversion'; did you mean '--version'?
gcc: fatal error: no input files
compilation terminated.
configure:3909: $? = 1
configure:3929: checking whether the C compiler works
configure:3951: gcc conftest.c >&5
/lib/../lib64/crt1.o: file not recognized: File truncated
collect2: error: ld returned 1 exit status
configure:3955: $? = 1
configure:3993: result: no
configure: failed program was:
| /* confdefs.h */
| #define _GNU_SOURCE 1
| #define _NETBSD_SOURCE 1
| #define __BSD_VISIBLE 1
| #define _DARWIN_C_SOURCE 1
| #define _PYTHONFRAMEWORK ""
| #define _XOPEN_SOURCE 700
| #define _XOPEN_SOURCE_EXTENDED 1
| #define _POSIX_C_SOURCE 200809L
| /* end confdefs.h. */
|
| int
| main ()
| {
|
| ;
| return 0;
| }
configure:3998: error: in `/home/user1/Python-3.9.6':
configure:4000: error: C compiler cannot create executables
See `config.log' for more details
## ---------------- ##
## Cache variables. ##
## ---------------- ##
ac_cv_build=x86_64-pc-linux-gnu
ac_cv_env_CC_set=
ac_cv_env_CC_value=
ac_cv_env_CFLAGS_set=
ac_cv_env_CFLAGS_value=
ac_cv_env_CPPFLAGS_set=
ac_cv_env_CPPFLAGS_value=
ac_cv_env_CPP_set=
ac_cv_env_CPP_value=
ac_cv_env_LDFLAGS_set=
ac_cv_env_LDFLAGS_value=
ac_cv_env_LIBS_set=
ac_cv_env_LIBS_value=
ac_cv_env_MACHDEP_set=
ac_cv_env_MACHDEP_value=
ac_cv_env_PKG_CONFIG_LIBDIR_set=
ac_cv_env_PKG_CONFIG_LIBDIR_value=
ac_cv_env_PKG_CONFIG_PATH_set=set
ac_cv_env_PKG_CONFIG_PATH_value=/opt/rh/devtoolset-8/root/usr/lib64/pkgconfig
ac_cv_env_PKG_CONFIG_set=
ac_cv_env_PKG_CONFIG_value=
ac_cv_env_PROFILE_TASK_set=
ac_cv_env_PROFILE_TASK_value=
ac_cv_env_build_alias_set=
ac_cv_env_build_alias_value=
ac_cv_env_host_alias_set=
ac_cv_env_host_alias_value=
ac_cv_env_target_alias_set=
ac_cv_env_target_alias_value=
ac_cv_host=x86_64-pc-linux-gnu
ac_cv_prog_PYTHON_FOR_REGEN=python3
ac_cv_prog_ac_ct_CC=gcc
## ----------------- ##
## Output variables. ##
## ----------------- ##
ABIFLAGS=''
ALT_SOABI=''
AR=''
ARCH_RUN_32BIT=''
ARFLAGS=''
BASECFLAGS=''
BASECPPFLAGS=''
BINLIBDEST=''
BLDLIBRARY=''
BLDSHARED=''
BUILDEXEEXT=''
CC='gcc'
CCSHARED=''
CFLAGS=''
CFLAGSFORSHARED=''
CFLAGS_ALIASING=''
CFLAGS_NODIST=''
CONFIGURE_MACOSX_DEPLOYMENT_TARGET=''
CONFIG_ARGS=' '\''--enable-optimizations'\'' '\''PKG_CONFIG_PATH=/opt/rh/devtoolset-8/root/usr/lib64/pkgconfig'\'''
CPP=''
CPPFLAGS=''
CXX=''
DEFS=''
DEF_MAKE_ALL_RULE=''
DEF_MAKE_RULE=''
DFLAGS=''
DLINCLDIR=''
DLLLIBRARY=''
DTRACE=''
DTRACE_HEADERS=''
DTRACE_OBJS=''
DYNLOADFILE=''
ECHO_C=''
ECHO_N='-n'
ECHO_T=''
EGREP=''
ENSUREPIP=''
EXEEXT=''
EXPORTSFROM=''
EXPORTSYMS=''
EXPORT_MACOSX_DEPLOYMENT_TARGET='#'
EXT_SUFFIX=''
FRAMEWORKALTINSTALLFIRST=''
FRAMEWORKALTINSTALLLAST=''
FRAMEWORKINSTALLAPPSPREFIX=''
FRAMEWORKINSTALLFIRST=''
FRAMEWORKINSTALLLAST=''
FRAMEWORKPYTHONW=''
FRAMEWORKUNIXTOOLSPREFIX='/usr/local'
GITBRANCH=''
GITTAG=''
GITVERSION=''
GNULD=''
GREP=''
HAS_GIT='no-repository'
HAVE_GETHOSTBYNAME=''
HAVE_GETHOSTBYNAME_R=''
HAVE_GETHOSTBYNAME_R_3_ARG=''
HAVE_GETHOSTBYNAME_R_5_ARG=''
HAVE_GETHOSTBYNAME_R_6_ARG=''
INSTALL_DATA=''
INSTALL_PROGRAM=''
INSTALL_SCRIPT=''
INSTSONAME=''
LDCXXSHARED=''
LDFLAGS=''
LDFLAGS_NODIST=''
LDLIBRARY=''
LDLIBRARYDIR=''
LDSHARED=''
LDVERSION=''
LIBC=''
LIBFFI_INCLUDEDIR=''
LIBM=''
LIBOBJS=''
LIBPL=''
LIBPYTHON=''
LIBRARY=''
LIBS=''
LIBTOOL_CRUFT=''
LINKCC=''
LINKFORSHARED=''
LIPO_32BIT_FLAGS=''
LIPO_INTEL64_FLAGS=''
LLVM_AR=''
LLVM_AR_FOUND=''
LLVM_PROFDATA=''
LLVM_PROF_ERR=''
LLVM_PROF_FILE=''
LLVM_PROF_FOUND=''
LLVM_PROF_MERGER=''
LN=''
LTLIBOBJS=''
MACHDEP='linux'
MACHDEP_OBJS=''
MAINCC=''
MKDIR_P=''
MULTIARCH=''
MULTIARCH_CPPFLAGS=''
NO_AS_NEEDED=''
OBJEXT=''
OPENSSL_INCLUDES=''
OPENSSL_LDFLAGS=''
OPENSSL_LIBS=''
OPT=''
OTHER_LIBTOOL_OPT=''
PACKAGE_BUGREPORT='https://bugs.python.org/'
PACKAGE_NAME='python'
PACKAGE_STRING='python 3.9'
PACKAGE_TARNAME='python'
PACKAGE_URL=''
PACKAGE_VERSION='3.9'
PATH_SEPARATOR=':'
PGO_PROF_GEN_FLAG=''
PGO_PROF_USE_FLAG=''
PKG_CONFIG=''
PKG_CONFIG_LIBDIR=''
PKG_CONFIG_PATH='/opt/rh/devtoolset-8/root/usr/lib64/pkgconfig'
PLATFORM_TRIPLET=''
PLATLIBDIR=''
PROFILE_TASK=''
PY3LIBRARY=''
PYTHONFRAMEWORK=''
PYTHONFRAMEWORKDIR='no-framework'
PYTHONFRAMEWORKIDENTIFIER='org.python.python'
PYTHONFRAMEWORKINSTALLDIR=''
PYTHONFRAMEWORKPREFIX=''
PYTHON_FOR_BUILD='./$(BUILDPYTHON) -E'
PYTHON_FOR_REGEN='python3'
PY_ENABLE_SHARED=''
READELF=''
RUNSHARED=''
SED=''
SHELL='/bin/sh'
SHLIBS=''
SHLIB_SUFFIX=''
SOABI=''
SOVERSION='1.0'
SRCDIRS=''
TCLTK_INCLUDES=''
TCLTK_LIBS=''
THREADHEADERS=''
TRUE=''
TZPATH=''
UNIVERSALSDK=''
UNIVERSAL_ARCH_FLAGS=''
VERSION='3.9'
_PYTHON_HOST_PLATFORM=''
ac_ct_AR=''
ac_ct_CC='gcc'
ac_ct_CXX=''
ac_ct_READELF=''
bindir='${exec_prefix}/bin'
build='x86_64-pc-linux-gnu'
build_alias=''
build_cpu='x86_64'
build_os='linux-gnu'
build_vendor='pc'
datadir='${datarootdir}'
datarootdir='${prefix}/share'
docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
dvidir='${docdir}'
exec_prefix='NONE'
host='x86_64-pc-linux-gnu'
host_alias=''
host_cpu='x86_64'
host_os='linux-gnu'
host_vendor='pc'
htmldir='${docdir}'
includedir='${prefix}/include'
infodir='${datarootdir}/info'
libdir='${exec_prefix}/lib'
libexecdir='${exec_prefix}/libexec'
localedir='${datarootdir}/locale'
localstatedir='${prefix}/var'
mandir='${datarootdir}/man'
oldincludedir='/usr/include'
pdfdir='${docdir}'
prefix='NONE'
program_transform_name='s,x,x,'
psdir='${docdir}'
sbindir='${exec_prefix}/sbin'
sharedstatedir='${prefix}/com'
sysconfdir='${prefix}/etc'
target_alias=''
## ----------- ##
## confdefs.h. ##
## ----------- ##
/* confdefs.h */
#define _GNU_SOURCE 1
#define _NETBSD_SOURCE 1
#define __BSD_VISIBLE 1
#define _DARWIN_C_SOURCE 1
#define _PYTHONFRAMEWORK ""
#define _XOPEN_SOURCE 700
#define _XOPEN_SOURCE_EXTENDED 1
#define _POSIX_C_SOURCE 200809L
configure: exit 77
Is there something that I need to install within my Linux (Centos 7) to get this to work?
My current Python Version is 3.6.8
Im translating a model done on weka to python-weka-wrapper3 and i dont know how to an evaluator and search options on attributeselectedclassifier.
This is the model on weka:
weka.classifiers.meta.AttributeSelectedClassifier -E "weka.attributeSelection.CfsSubsetEval -P 1 -E 1" -S "weka.attributeSelection.GreedyStepwise -B -T -1.7976931348623157E308 -N -1 -num-slots 1" -W weka.classifiers.meta.MultiSearch -- -E FM -search "weka.core.setupgenerator.MathParameter -property classifier.classifier.classifier.numOfBoostingIterations -min 5.0 -max 50.0 -step 1.0 -base 10.0 -expression I" -class-label 1 -algorithm "weka.classifiers.meta.multisearch.DefaultSearch -sample-size 100.0 -initial-folds 2 -subsequent-folds 10 -initial-test-set . -subsequent-test-set . -num-slots 1" -log-file /Applications/weka-3-8-3 -S 1 -W weka.classifiers.meta.Bagging -- -P 100 -S 1 -num-slots 1 -I 100 -W weka.classifiers.meta.FilteredClassifier -- -F "weka.filters.supervised.instance.SMOTE -C 0 -K 3 -P 250.0 -S 1" -S 1 -W weka.classifiers.meta.CostSensitiveClassifier -- -cost-matrix "[0.0 1.0; 1.0 0.0]" -S 1 -W weka.classifiers.trees.ADTree -- -B 10 -E -3 -S 1
and I have this right now:
base = Classifier(classname="weka.classifiers.trees.ADTree", options=["-B", "10", "-E", "-3", "-S", "1"])
cls = SingleClassifierEnhancer(classname="weka.classifiers.meta.CostSensitiveClassifier",
options =["-cost-matrix", "[0.0 1.0; 1.0 0.0]", "-S", "1"])
cls.classifier = base
smote = Filter(classname="weka.filters.supervised.instance.SMOTE", options=["-C", "0", "-K", "3", "-P", "250.0", "-S", "1"])
fc = FilteredClassifier()
fc.filter = smote
fc.classifier = cls
bagging_cls = Classifier(classname="weka.classifiers.meta.Bagging",
options=["-P", "100", "-S", "1", "-num-slots", "1", "-I", "100"])
bagging_cls.classifier = fc
multisearch_cls = MultiSearch(
options = ["-S", "1"])
multisearch_cls.evaluation = "FM"
multisearch_cls.log_file = "/home/pablo/Escritorio/TFG/OUTPUT.txt"
multisearch_cls.search = ["-sample-size", "100", "-initial-folds", "2", "-subsequent-folds", "10",
"-initial-test-set", ".", "-subsequent-test-set", ".", "-num-slots", "1"]
mparam = MathParameter()
mparam.prop = "numOfBoostingIterations"
mparam.minimum = 5.0
mparam.maximum = 50.0
mparam.step = 1.0
mparam.base = 10.0
mparam.expression = "I"
multisearch_cls.parameters = [mparam]
multisearch_cls.classifier = bagging_cls
AttS_cls = AttributeSelectedClassifier()
AttS_cls.evaluator = "weka.attributeSelection.CfsSubsetEval -P 1 -E 1"
AttS_cls.search = "weka.attributeSelection.GreedyStepwise -B -T -1.7976931348623157E308 -N -1 -num-slots 1"
AttS_cls.classifier = multisearch_cls
train, test = data_modelos_1_2.train_test_split(70.0, Random(1))
AttS_cls.build_classifier(train)
evl = Evaluation(train)
evl.crossvalidate_model(AttS_cls, test, 10, Random(1))
print(AttS_cls)
#graph.plot_dot_graph(AttS_cls.graph)
print("")
print("=== Setup ===")
print("Classifier: " + AttS_cls.to_commandline())
print("Dataset: ")
print(test.relationname)
print("")
print(evl.summary("=== " + str(10) + " -fold Cross-Validation ==="))
print(evl.class_details())
plcls.plot_roc(evl, class_index=[0, 1], wait=True)
but when I do
AttS_cls.evaluator = "weka.attributeSelection.CfsSubsetEval -P 1 -E 1"
AttS_cls.search = "weka.attributeSelection.GreedyStepwise -B -T -1.7976931348623157E308 -N -1 -num-slots 1"
it reach me this error:
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
/tmp/ipykernel_40724/2750622902.py in <module>
30
31 AttS_cls = AttributeSelectedClassifier()
---> 32 AttS_cls.search = "weka.attributeSelection.GreedyStepwise"
33 AttS_cls.classifier = multisearch_cls
34
/usr/local/lib/python3.8/dist-packages/weka/classifiers.py in search(self, search)
435 :type search: ASSearch
436 """
--> 437 javabridge.call(self.jobject, "setSearch", "(Lweka/attributeSelection/ASSearch;)V", search.jobject)
438
439
AttributeError: 'str' object has no attribute 'jobject'
I understand that I have to set them as objects because it raise this error because i try to set them as strings but I dont know how.
You need to instantiate ASSearch and ASEvaluation objects. If you have command-lines, you can use the from_commandline helper method like this:
from weka.core.classes import from_commandline, get_classname
from weka.attribute_selection import ASSearch
from weka.attribute_selection import ASEvaluation
search = from_commandline('weka.attributeSelection.GreedyStepwise -B -T -1.7976931348623157E308 -N -1 -num-slots 1', classname=get_classname(ASSearch))
evaluation = from_commandline('weka.attributeSelection.CfsSubsetEval -P 1 -E 1', classname=get_classname(ASEvaluation))
The second argument of the from_commandline method is the classname of the wrapper that you want to use instead of OptionHandler. For simplicity, I import the correct wrappers and then use the get_classname method to return the dot notation of the wrapper's class. That way I can avoid accidental typos in the classname strings.
Also, by using single quotes, you won't have to worry about Weka's quotes in the command-lines and you can just use the Weka command-line string verbatim.
You can also use the same approach for instantiating the AttributeSelectedClassifier wrapper itself, without having to go through instantiating search and evaluation separately:
from weka.core.classes import from_commandline, get_classname
from weka.classifiers import AttributeSelectedClassifier
cls = from_commandline('weka.classifiers.meta.AttributeSelectedClassifier -E "weka.attributeSelection.CfsSubsetEval -P 1 -E 1" -S "weka.attributeSelection.GreedyStepwise -B -T -1.7976931348623157E308 -N -1 -num-slots 1" -W weka.classifiers.meta.MultiSearch -- -E FM -search "weka.core.setupgenerator.MathParameter -property classifier.classifier.classifier.numOfBoostingIterations -min 5.0 -max 50.0 -step 1.0 -base 10.0 -expression I" -class-label 1 -algorithm "weka.classifiers.meta.multisearch.DefaultSearch -sample-size 100.0 -initial-folds 2 -subsequent-folds 10 -initial-test-set . -subsequent-test-set . -num-slots 1" -log-file /Applications/weka-3-8-3 -S 1 -W weka.classifiers.meta.Bagging -- -P 100 -S 1 -num-slots 1 -I 100 -W weka.classifiers.meta.FilteredClassifier -- -F "weka.filters.supervised.instance.SMOTE -C 0 -K 3 -P 250.0 -S 1" -S 1 -W weka.classifiers.meta.CostSensitiveClassifier -- -cost-matrix "[0.0 1.0; 1.0 0.0]" -S 1 -W weka.classifiers.trees.ADTree -- -B 10 -E -3 -S 1', get_classname(AttributeSelectedClassifier))
Closed. This question needs to be more focused. It is not currently accepting answers.
Want to improve this question? Update the question so it focuses on one problem only by editing this post.
Closed 1 year ago.
Improve this question
I need to merge two sets of files together. I don't mind what language is used as long as it works.
For example:
I have a folder with both sets of files
N-4A-2A_S135_L001_R1_001.fastq.gz
N-4A-2A_S135_L001_R2_001.fastq.gz
N-4A-2A_S135_L001_R1_001-2.fastq.gz
N-4A-2A_S135_L001_R2_001-2.fastq.gz
N-4-Bmp-1_S20_L001_R1_001.fastq.gz
N-4-Bmp-1_S20_L001_R2_001.fastq.gz
N-4-Bmp-1_S20_L001_R1_001-2.fastq.gz
N-4-Bmp-1_S20_L001_R2_001-2.fastq.gz
I need them merged like this:
N-4A-2A-S135-L001_R1_001.fastq.gz + N-4A-2A-S135-L001_R1_001-2.fastq.gz > N-4A-2A-S135-L001_R1_001-cat.fastq.gz
N-4A-2A-S135-L001_R2_001.fastq.gz + N-4A-2A-S135-L001_R2_001-2.fastq.gz > N-4A-2A-S135-L001_R2_001-cat.fastq.gz
N-4-Bmp-1_S20_L001_R1_001.fastq.gz + N-4-Bmp-1_S20_L001_R1_001-2.fastq.gz > N-4-Bmp-1_S20_L001_R1_001-cat.fastq.gz
N-4-Bmp-1_S20_L001_R2_001.fastq.gz + N-4-Bmp-1_S20_L001_R2_001-2.fastq.gz > N-4-Bmp-1_S20_L001_R2_001-cat.fastq.gz
I have a total of ~180 files so any way to automate this would be amazing.
They all follow the same format of:
(sample name) + "R1_001" or "R2_001" + "" or "-2" + .fastq.gz
According to example you provided, your file names look sortable. If this is not the case, this will not work and this answer will be invalid.
Idea of the following snippet is really simple - sort all the filenames and then join adjacent tuples.
This should work in bash:
#!/bin/bash
files=( $( find . -type f | sort -r) )
files_num=${#files[#]}
for (( i=0 ; i < ${files_num} ; i=i+2 )); do
f1="${files[i]}"
f2="${files[i+1]}"
fname=$(basename -s '.gz' "$f1")".cat.gz"
echo "$f1 and $f2 -> $fname"
cat "$f1" "$f2" > "$fname"
done
or simpler version in ZShell (zsh)
#!/bin/zsh
files=( $( find . -type f | sort -r) )
for f1 f2 in $files; do
fname=$(basename -s '.gz' "$f1")".cat.gz"
echo "$f1 and $f2 -> $fname"
cat "$f1" "$f2" > "$fname"
done
You did say any language, so ... I prefer perl
#!/usr/bin/perl
# concatenate files
master(#ARGV);
exit(0);
# master -- master control
sub master
{
my(#argv) = #_;
my($tail,#tails);
local(%tails);
# scan argv for -opt
while (1) {
my($opt) = $argv[0];
last unless (defined($opt));
last unless ($opt =~ s/^-/opt_/);
$$opt = 1;
shift(#argv);
}
# load up the current directory
#tails = dirload(".");
foreach $tail (#tails) {
$tails{$tail} = 0;
}
# look for the base name of a file
foreach $tail (#tails) {
docat($tail)
if ($tail =~ /R[12]_001\.fastq\.gz$/);
}
# default mode is "dry run"
unless ($opt_go) {
printf("\n");
printf("rerun with -go to actually do it\n");
}
}
# docat -- process a pairing
sub docat
{
my($base) = #_;
my($tail);
my($out);
my($cmd);
my($code);
# all commands are joining just two files
$tail = $base;
$tail =~ s/\.fastq/-2.fastq/;
# to an output file
$out = $base;
$out =~ s/\.fastq/-cat.fastq/;
$cmd = "cat $base $tail > $out";
if ($opt_v) {
printf("\n");
printf("IN1: %s\n",$base);
printf("IN2: %s\n",$tail);
printf("OUT: %s\n",$out);
}
else {
printf("%s\n",$cmd);
}
die("docat: duplicate IN1\n")
if ($tails{$base});
$tails{$base} = 1;
die("docat: duplicate IN2\n")
if ($tails{$tail});
$tails{$tail} = 1;
die("docat: duplicate OUT\n")
if ($tails{$out});
$tails{$out} = 1;
{
last unless ($opt_go);
# execute the command and get error code
system($cmd);
$code = $? >> 8;
exit($code) if ($code);
}
}
# dirload -- get list of files in a directory
sub dirload
{
my($dir) = #_;
my($xf);
my($tail);
my(#tails);
# open the directory
opendir($xf,$dir) or
die("dirload: unable to open '$dir' -- $!\n");
# get list of files in the directory excluding "." and ".."
while ($tail = readdir($xf)) {
next if ($tail eq ".");
next if ($tail eq "..");
push(#tails,$tail);
}
closedir($xf);
#tails = sort(#tails);
#tails;
}
Here's the program output with the -v option:
IN1: N-4-Bmp-1_S20_L001_R1_001.fastq.gz
IN2: N-4-Bmp-1_S20_L001_R1_001-2.fastq.gz
OUT: N-4-Bmp-1_S20_L001_R1_001-cat.fastq.gz
IN1: N-4-Bmp-1_S20_L001_R2_001.fastq.gz
IN2: N-4-Bmp-1_S20_L001_R2_001-2.fastq.gz
OUT: N-4-Bmp-1_S20_L001_R2_001-cat.fastq.gz
IN1: N-4A-2A_S135_L001_R1_001.fastq.gz
IN2: N-4A-2A_S135_L001_R1_001-2.fastq.gz
OUT: N-4A-2A_S135_L001_R1_001-cat.fastq.gz
IN1: N-4A-2A_S135_L001_R2_001.fastq.gz
IN2: N-4A-2A_S135_L001_R2_001-2.fastq.gz
OUT: N-4A-2A_S135_L001_R2_001-cat.fastq.gz
rerun with -go to actually do it
Running the same python script using python3 or through an embedded interpreter using libpython3 gives different execution times.
$ time PYTHONPATH=. ./simple
real 0m6,201s
user 1m3,680s
sys 0m0,212s
$ time PYTHONPATH=. python3 -c 'import test; test.run()'
real 0m5,193s
user 0m53,349s
sys 0m0,164s
(removing the content of __pycache__ between runs does not seem to have an impact)
Currently, calling python3 with the script is faster; on my actual use case the factor is 1.5 faster, compared to the same script ran from within an embedded interpreter.
I would like to (1) understand where does the difference come from and (2) if it is possible to have the same performance using an embedded interpreter? (using e.g. cython is currently not an option).
Code
simple.cpp
#include <Python.h>
int main()
{
Py_Initialize();
const char* pythonScript = "import test; test.run()";
int result = PyRun_SimpleString(pythonScript);
Py_Finalize();
return result;
}
Compilation:
g++ -std=c++11 -fPIC $(python3-config --cflags) simple.cpp \
$(python3-config --ldflags) -o simple
test.py
import sys
sys.stdout = open('output.bin', 'bw')
import mandel
def run():
mandel.mandelbrot(4096)
mandel.py
Tweaked version from benchmarks-game's Mandlebrot (see License)
from contextlib import closing
from itertools import islice
from os import cpu_count
from sys import stdout
def pixels(y, n, abs):
range7 = bytearray(range(7))
pixel_bits = bytearray(128 >> pos for pos in range(8))
c1 = 2. / float(n)
c0 = -1.5 + 1j * y * c1 - 1j
x = 0
while True:
pixel = 0
c = x * c1 + c0
for pixel_bit in pixel_bits:
z = c
for _ in range7:
for _ in range7:
z = z * z + c
if abs(z) >= 2.: break
else:
pixel += pixel_bit
c += c1
yield pixel
x += 8
def compute_row(p):
y, n = p
result = bytearray(islice(pixels(y, n, abs), (n + 7) // 8))
result[-1] &= 0xff << (8 - n % 8)
return y, result
def ordered_rows(rows, n):
order = [None] * n
i = 0
j = n
while i < len(order):
if j > 0:
row = next(rows)
order[row[0]] = row
j -= 1
if order[i]:
yield order[i]
order[i] = None
i += 1
def compute_rows(n, f):
row_jobs = ((y, n) for y in range(n))
if cpu_count() < 2:
yield from map(f, row_jobs)
else:
from multiprocessing import Pool
with Pool() as pool:
unordered_rows = pool.imap_unordered(f, row_jobs)
yield from ordered_rows(unordered_rows, n)
def mandelbrot(n):
write = stdout.write
with closing(compute_rows(n, compute_row)) as rows:
write("P4\n{0} {0}\n".format(n).encode())
for row in rows:
write(row[1])
So apparently the time difference comes from either linking with libpython statically vs. dynamically. In a Makefile sitting next to python.c (from the reference implementation), the following builds a statically linked version of the interpreter:
snake: python.c
g++ \
-I/usr/include/python3.6m \
-pthread \
-specs=/usr/share/dpkg/no-pie-link.specs \
-specs=/usr/share/dpkg/no-pie-compile.specs \
\
-Wall \
-Wformat \
-Werror=format-security \
-Wno-unused-result \
-Wsign-compare \
-DNDEBUG \
-g \
-fwrapv \
-fstack-protector \
-O3 \
\
-Xlinker -export-dynamic \
-Wl,-Bsymbolic-functions \
-Wl,-z,relro \
-Wl,-O1 \
python.c \
/usr/lib/python3.6/config-3.6m-x86_64-linux-gnu/libpython3.6m.a \
-lexpat \
-lpthread \
-ldl \
-lutil \
-lexpat \
-L/usr/lib \
-lz \
-lm \
-o $#
Changing the line /usr/lib/.../libpython3.6m.a with -llibpython3.6m builds the version that ends up being slower (also need -L/usr/lib/python3.6/config-3.6m-x86_64-linux-gnu)
Epilog
The difference in speed exists but is not the full answer to my original problem; in practice the "slower" interpreter was executed under a specific LD_PRELOAD environment which changed how system time functions behaves in a way that messed up with cProfile.
I have a project with many subdirectories and types of files (python, c++, configuration files, images, etc).
When I use SCons env.Package like this:
env.Package(
NAME = 'isid',
VERSION = '1.0',
PACKAGEVERSION = '11',
PACKAGETYPE = 'rpm',
LICENSE = 'gpl',
SUMMARY = 'just a test',
DESCRIPTION = 'the most test app in the world',
X_RPM_GROUP = 'Application/isid',
SOURCE_URL = 'http://isid.com/versions/isid-1.0.11.tar.gz',
)
I get everything in isid-1.0.11.tar.gz except for the h files.
This automatically leads to build errors in ./isid-1.0.11 that stops rpmbuild from running.
EDIT
My project is split into few subdirectories.
In each I have SConscript that starts with these lines, or similar, depending on the includes it needs:
# import all variables
Import('*')
include = Dir([
'../inc/',
])
local_env = env.Clone( CPPPATH = include )
SConstruct just defines the variables and calls SConscript() on each subdirectory.
The call to Package is done in SConstruct, so I guess SCons indeed does not know the dependencies.
snippet of
# scons --tree=prune:
scons: Reading SConscript files ...
scons: done reading SConscript files.
scons: Building targets ...
scons: `.' is up to date.
+-.
+-SConstruct
+-correlator
| +-correlator/SConscript
| +-correlator/collector
| +-correlator/correlation_command
| | +-correlator/correlation_command/app_command_device.cpp
| | +-correlator/correlation_command/app_command_device.h
| | +-correlator/correlation_command/app_command_device.o
| | | +-correlator/correlation_command/app_command_device.cpp
| | | +-correlator/correlation_command/app_command_device.h
| | | +-correlator/entity/entity.h
| | | +-infra/inc/app_command/app_command.h
| | | +-/bin/g++
| | +-correlator/correlation_command/app_command_event.cpp
| | +-correlator/correlation_command/app_command_event.h
EDIT #2
Here is a complete, minimal, example that produces the same problem.
To reproduce the problem, run:
scons pack=1.0 ver=1
files:
SConstruct.
app1/SConscript
app1/main.cpp
app1/inc.h
Listing
SConstruct
1
2 # main file
3
4 env = Environment(tools=['default', 'packaging'])
5
6 Export( 'env' )
7
8 flags = [
9 '-Wall',
10 '-Werror',
11 '-g',
12 '-ggdb3',
13 '-gdwarf-2',
14 '-std=c++11',
15 ]
16
17 env.Append( CPPFLAGS = flags )
18
19
20 scripts = []
21
22 Sapp1 = 'app1/SConscript'
23 scripts.append( Sapp1 )
24
25
26 env.SConscript( scripts )
27
28 pack = ARGUMENTS.get('pack', '')
29 ver = ARGUMENTS.get('ver', '99' )
30 if pack:
31 env.Package(
32 NAME = 'app1',
33 VERSION = pack,
34 PACKAGEVERSION = ver,
35 PACKAGETYPE = 'rpm',
36 LICENSE = 'private',
37 SUMMARY = 'exampe app #1',
38 DESCRIPTION = 'the most powerfull exampe #1',
39 X_RPM_GROUP = 'Application/app1',
40 SOURCE_URL = 'http://example.com/1/app1-1.0.1.tar.gz',
41 )
42
app1/SConscript
1
2 # import all variables
3 Import('*')
4
5 # add specific include directory
6
7 include = Dir( [
8 '.',
9 ])
10
11 local_env = env.Clone( CPPPATH = include )
12
13 # define sources
14 sources = [
15 'main.cpp',
16 ]
17
18 libs = [
19 ]
20
21 main_name = 'app1',
22
23 main_obj = local_env.Program( target = main_name, source = sources, LIBS = libs )
24
25 # install
26 install_dir = '/opt/rf/app1'
27 install_files = [ main_obj ]
28
29 local_env.Install( dir = install_dir, source = install_files )
30 local_env.Command( install_dir, install_files, "chown -R rf:rfids $TARGET" )
31
32
33 local_env.Alias( 'install', install_dir )
app1/main.cpp
1
2 #include "inc.h"
3
4
5 int main()
6 {
7 int l = g;
8
9 return l;
10 }
app1/inc.h
1
2 int g = 100;
output:
# scons pack=1.0 ver=1
scons: Reading SConscript files ...
scons: done reading SConscript files.
scons: Building targets ...
g++ -o app1/main.o -c -Wall -Werror -g -ggdb3 -gdwarf-2 -std=c++11 -Iapp1 app1/main.cpp
g++ -o app1/app1 app1/main.o
LC_ALL=C rpmbuild -ta /home/ran/work/rpmexample/app1-1.0.1.tar.gz
scons: *** [app1-1.0-1.src.rpm] app1-1.0-1.src.rpm: Executing(%prep): /bin/sh -e /var/tmp/rpm-tmp.wU9lDZ
+ umask 022
+ cd /home/ran/work/rpmexample/rpmtemp/BUILD
+ '[' -n /home/ran/work/rpmexample/rpmtemp/BUILDROOT/app1-1.0-1.x86_64 -a /home/ran/work/rpmexample/rpmtemp/BUILDROOT/app1-1.0-1.x86_64 '!=' / ']'
+ rm -rf /home/ran/work/rpmexample/rpmtemp/BUILDROOT/app1-1.0-1.x86_64
+ cd /home/ran/work/rpmexample/rpmtemp/BUILD
+ rm -rf app1-1.0
+ /usr/bin/gzip -dc /home/ran/work/rpmexample/app1-1.0.1.tar.gz
+ /usr/bin/tar -xf -
+ STATUS=0
+ '[' 0 -ne 0 ']'
+ cd app1-1.0
+ /usr/bin/chmod -Rf a+rX,u+w,g-w,o-w .
+ exit 0
Executing(%build): /bin/sh -e /var/tmp/rpm-tmp.gVaX4j
+ umask 022
+ cd /home/ran/work/rpmexample/rpmtemp/BUILD
+ cd app1-1.0
+ '[' '!' -e /home/ran/work/rpmexample/rpmtemp/BUILDROOT/app1-1.0-1.x86_64 -a /home/ran/work/rpmexample/rpmtemp/BUILDROOT/app1-1.0-1.x86_64 '!=' / ']'
+ mkdir /home/ran/work/rpmexample/rpmtemp/BUILDROOT/app1-1.0-1.x86_64
+ exit 0
Executing(%install): /bin/sh -e /var/tmp/rpm-tmp.JWAdxE
+ umask 022
+ cd /home/ran/work/rpmexample/rpmtemp/BUILD
+ '[' /home/ran/work/rpmexample/rpmtemp/BUILDROOT/app1-1.0-1.x86_64 '!=' / ']'
+ rm -rf /home/ran/work/rpmexample/rpmtemp/BUILDROOT/app1-1.0-1.x86_64
++ dirname /home/ran/work/rpmexample/rpmtemp/BUILDROOT/app1-1.0-1.x86_64
+ mkdir -p /home/ran/work/rpmexample/rpmtemp/BUILDROOT
+ mkdir /home/ran/work/rpmexample/rpmtemp/BUILDROOT/app1-1.0-1.x86_64
+ cd app1-1.0
+ scons --install-sandbox=/home/ran/work/rpmexample/rpmtemp/BUILDROOT/app1-1.0-1.x86_64 /home/ran/work/rpmexample/rpmtemp/BUILDROOT/app1-1.0-1.x86_64
scons: Reading SConscript files ...
scons: done reading SConscript files.
scons: Building targets ...
g++ -o app1/main.o -c -Wall -Werror -g -ggdb3 -gdwarf-2 -std=c++11 -Iapp1 app1/main.cpp
app1/main.cpp:2:17: fatal error: inc.h: No such file or directory
#include "inc.h"
^
compilation terminated.
scons: *** [app1/main.o] Error 1
scons: building terminated because of errors.
error: Bad exit status from /var/tmp/rpm-tmp.JWAdxE (%install)
RPM build errors:
Bad exit status from /var/tmp/rpm-tmp.JWAdxE (%install)
scons: building terminated because of errors.