Fundamental issue with my Gstreamer pipeline logic in Python. Any advice? - python

I have the following piece of code:
import pi
from pi.becore import ScarlettConfig
from recorder import Recorder
from brain import Brain
import os
import json
import tempfile
#import sys
import pygtk
pygtk.require('2.0')
import gtk
import gobject
import pygst
pygst.require('0.10')
gobject.threads_init()
import gst
scarlett_config=ScarlettConfig()
class Listener:
def __init__(self, gobject, gst):
self.failed = 0
# scarlettPi ( mmj2f I added this after attempting the scarlettPi change )
self.recording = tempfile.mktemp(suffix='pi.wav')
self.pipeline = gst.Pipeline("mypipeline")
# alsasrc - we'll use this to connect our ps3 eye and record
gst.debug("Adding alsasrc")
self.source = gst.element_factory_make("alsasrc", "alsasrc")
self.source.set_property("device", scarlett_config.gimmie('audio_input_device'))
self.pipeline.add(self.source)
# add a queue to allow pocketsphinx to recognize more data
gst.debug("Adding encoding queue")
self.qone = gst.element_factory_make("queue", "qone")
self.pipeline.add(self.qone)
# audio convert
gst.debug("Adding encoding audioconvert")
self.recfileconvert = gst.element_factory_make("audioconvert", "recfileconvert")
self.pipeline.add(self.recfileconvert)
# resample the wav
gst.debug("Adding encoding audioresample")
self.resample = gst.element_factory_make("audioresample", "resample")
self.pipeline.add(self.resample)
# Add our tee
gst.debug("Adding tee")
self.rectee = gst.element_factory_make("tee", "rectee")
self.pipeline.add(self.rectee)
# taken from: http://www.jonobacon.org/files/recgui-works.py
# link tee to alsasrc
gst.debug("Linking alsasrc to tee")
self.source.link(self.rectee)
gst.debug("Adding encoding queue")
self.qtwo = gst.element_factory_make("queue", "qtwo")
self.pipeline.add(self.qtwo)
gst.debug("Adding vader element")
self.vader = gst.element_factory_make("vader","vader")
self.vader.set_property("auto-threshold",False)
self.pipeline.add(self.vader)
gst.debug("Adding pocketsphinx element")
self.pocketsphinx = gst.element_factory_make("pocketsphinx","listener")
print "Pocketsphinx: "
print dir( self.pocketsphinx )
self.pocketsphinx.set_property("lm",scarlett_config.gimmie('LM'))
self.pocketsphinx.set_property("dict",scarlett_config.gimmie('DICT'))
self.pocketsphinx.set_property("hmm",scarlett_config.gimmie('HMM'))
gst.debug("Adding fakesink")
self.fakesink = gst.element_factory_make("fakesink", "fakesink")
self.fakesink.set_property("dump", True)
self.pipeline.add(self.fakesink)
# Linking fakesink to tee
# taken from: http://www.jonobacon.org/files/recgui-works.py
gst.debug("Linking fakesink to tee")
self.fakesink.link(self.rectee)
# creating valve now
gst.debug("Adding Valve element")
self.recording_valve = gst.element_factory_make('valve')
self.record_valve.set_property("drop",False)
self.pipeline.add(self.recording_valve)
# another queue
gst.debug("Adding encoding queue")
self.recording_valve_queue = gst.element_factory_make("queue", "recording_valve_queue")
self.pipeline.add(self.recording_valve_queue)
self.recording_valve.link (self.recording_valve_queue)
# adding wavenc element
gst.debug("Adding wavenc")
self.wavenc = gst.element_factory_make("wavenc", "wavenc")
self.pipeline.add(self.wavenc)
# adding filesink element
gst.debug("Adding filesink")
self.filesink = gst.element_factory_make("filesink", "filesink")
self.filesink.set_property("location", self.recording)
self.filesink.set_property("async", False)
self.pipeline.add(self.filesink)
# lets add the static pads now and link them?
# NOTE: Not sure if this is correct at all:
# self.rectee_pad = self.rectee.get_request_pad("src%d")
# print "Obtained request pad %s for audio branch."% self.rectee_pad.get_name()
self.rectee.get_request_pad('src%d').link(self.recording_valve_queue.get_pad('sink'))
self.rectee.get_request_pad('src%d').link(self.qtwo.get_pad('sink'))
# Original code going forward:
listener = self.pipeline.get_by_name('listener')
listener.connect('result', self.__result__)
listener.set_property('configured', True)
print "KEYWORDS WE'RE LOOKING FOR: " + scarlett_config.gimmie('ourkeywords')
......ETC..........
And I'm getting this error when I try to run my script:
pi#scarlettpi ~/dev/scarlettPi/scripts/pi $ pi
/usr/lib/python2.7/dist-packages/gtk-2.0/gtk/__init__.py:57: GtkWarning: could not open display
warnings.warn(str(e), _gtk.Warning)
INFO: cmd_ln.c(691): Parsing command line:
gst-pocketsphinx \
-samprate 8000 \
-cmn prior \
-fwdflat no \
-bestpath no \
-maxhmmpf 2000 \
-maxwpf 20
Current configuration:
[NAME] [DEFLT] [VALUE]
-agc none none
-agcthresh 2.0 2.000000e+00
-alpha 0.97 9.700000e-01
-ascale 20.0 2.000000e+01
-aw 1 1
-backtrace no no
-beam 1e-48 1.000000e-48
-bestpath no no
-bestpathlw 9.5 9.500000e+00
-bghist no no
-ceplen 13 13
-cmn current prior
-cmninit 8.0 8.0
-compallsen no no
-debug 0
-dict
-dictcase no no
-dither no no
-doublebw no no
-ds 1 1
-fdict
-feat 1s_c_d_dd 1s_c_d_dd
-featparams
-fillprob 1e-8 1.000000e-08
-frate 100 100
-fsg
-fsgusealtpron yes yes
-fsgusefiller yes yes
-fwdflat yes no
-fwdflatbeam 1e-64 1.000000e-64
-fwdflatefwid 4 4
-fwdflatlw 8.5 8.500000e+00
-fwdflatsfwin 25 25
-fwdflatwbeam 7e-29 7.000000e-29
-fwdtree yes yes
-hmm
-input_endian little little
-jsgf
-kdmaxbbi -1 -1
-kdmaxdepth 0 0
-kdtree
-latsize 5000 5000
-lda
-ldadim 0 0
-lextreedump 0 0
-lifter 0 0
-lm
-lmctl
-lmname default default
-logbase 1.0001 1.000100e+00
-logfn
-logspec no no
-lowerf 133.33334 1.333333e+02
-lpbeam 1e-40 1.000000e-40
-lponlybeam 7e-29 7.000000e-29
-lw 6.5 6.500000e+00
-maxhmmpf -1 2000
-maxnewoov 20 20
-maxwpf -1 20
-mdef
-mean
-mfclogdir
-min_endfr 0 0
-mixw
-mixwfloor 0.0000001 1.000000e-07
-mllr
-mmap yes yes
-ncep 13 13
-nfft 512 512
-nfilt 40 40
-nwpen 1.0 1.000000e+00
-pbeam 1e-48 1.000000e-48
-pip 1.0 1.000000e+00
-pl_beam 1e-10 1.000000e-10
-pl_pbeam 1e-5 1.000000e-05
-pl_window 0 0
-rawlogdir
-remove_dc no no
-round_filters yes yes
-samprate 16000 8.000000e+03
-seed -1 -1
-sendump
-senlogdir
-senmgau
-silprob 0.1 1.000000e-01
-smoothspec no no
-svspec
-tmat
-tmatfloor 0.0001 1.000000e-04
-topn 4 4
-topn_beam 0 0
-toprule
-transform legacy legacy
-unit_area yes yes
-upperf 6855.4976 6.855498e+03
-usewdphones no no
-uw 1.0 1.000000e+00
-var
-varfloor 0.0001 1.000000e-04
-varnorm no no
-verbose no no
-warp_params
-warp_type inverse_linear inverse_linear
-wbeam 7e-29 7.000000e-29
-wip 1e-4 1.000000e-04
-wlen 0.025625 2.562500e-02
Traceback (most recent call last):
File "/usr/local/bin/pi", line 5, in <module>
pkg_resources.run_script('pi==0.1.0', 'pi')
File "/usr/lib/python2.7/dist-packages/pkg_resources.py", line 499, in run_script
self.require(requires)[0].run_script(script_name, ns)
File "/usr/lib/python2.7/dist-packages/pkg_resources.py", line 1235, in run_script
execfile(script_filename, namespace, namespace)
File "/usr/local/lib/python2.7/dist-packages/pi-0.1.0-py2.7.egg/EGG-INFO/scripts/pi", line 38, in <module>
listener = Listener(gobject, gst)
File "/usr/local/lib/python2.7/dist-packages/pi-0.1.0-py2.7.egg/pi/listener.py", line 103, in __init__
self.pocketsphinx.link(self.rectee)
gst.LinkError: failed to link listener with rectee
I'm trying to do the following suggestion, but i'm not sure if i'm doing it correctly:
https://stackoverflow.com/a/18228933/814221
Can anyone help me understand what i'm doing wrong fundamentally?
Thanks!

You are missing the capsfilters.

Related

Can I render an openCV animation from a background thread in Python?

Can I render an openCV animation from a background thread in Python?
Here is my attempt:
import cv2
import numpy as np
from time import sleep
bitmap = np.zeros((512,512,3),np.uint8)
import threading
import time
def update_bitmap():
for i in range(512):
bitmap[i,i,:] = 128
sleep(1/32)
threading.Thread(target=update_bitmap).start()
def refresh_gui():
hz = 30
delta_t = 1 / hz
t = time.time()
while True:
sleep(0.001)
if time.time() > t+delta_t:
t += delta_t
cv2.imshow("Color Image", bitmap)
cv2.waitKey(1)
threading.Thread(target=refresh_gui).start()
try:
while True:
print('tick')
sleep(1)
except KeyboardInterrupt:
cv2.destroyAllWindows()
exit(0)
However it fails with:
> python test.py
tick
2021-03-14 17:01:37.474 python[38483:2301820] WARNING: NSWindow drag regions should only be invalidated on the Main Thread! This will throw an exception in the future. Called from (
0 AppKit 0x00007fff22c7347f -[NSWindow(NSWindow_Theme) _postWindowNeedsToResetDragMarginsUnlessPostingDisabled] + 352
1 AppKit 0x00007fff22c5e121 -[NSWindow _initContent:styleMask:backing:defer:contentView:] + 1375
2 AppKit 0x00007fff22c5dbbb -[NSWindow initWithContentRect:styleMask:backing:defer:] + 42
3 AppKit 0x00007fff22f681f4 -[NSWindow initWithContentRect:styleMask:backing:defer:screen:] + 52
4 cv2.cpython-37m-darwin.so 0x00000001084cc4f5 cvNamedWindow + 677
5 cv2.cpython-37m-darwin.so 0x00000001084cbdbc cvShowImage + 188
6 cv2.cpython-37m-darwin.so 0x00000001084ca286 _ZN2cv6imshowERKNSt3__112basic_stringIcNS0_11char_traitsIcEENS0_9allocatorIcEEEERKNS_11_InputArrayE + 230
7 cv2.cpython-37m-darwin.so 0x000000010749515e _ZL18pyopencv_cv_imshowP7_objectS0_S0_ + 302
8 python 0x0000000106fcd4e8 _PyMethodDef_RawFastCallKeywords + 392
9 python 0x0000000107109ce2 call_function + 306
10 python 0x0000000107106a83 _PyEval_EvalFrameDefault + 42243
11 python 0x0000000106fccbe5 function_code_fastcall + 117
12 python 0x0000000107107bf2 _PyEval_EvalFrameDefault + 46706
13 python 0x0000000106fccbe5 function_code_fastcall + 117
14 python 0x0000000107109c67 call_function + 183
15 python 0x00000001071069ed _PyEval_EvalFrameDefault + 42093
16 python 0x0000000106fccbe5 function_code_fastcall + 117
17 python 0x0000000107109c67 call_function + 183
18 python 0x00000001071069ed _PyEval_EvalFrameDefault + 42093
19 python 0x0000000106fccbe5 function_code_fastcall + 117
20 python 0x0000000106fd0002 method_call + 130
21 python 0x0000000106fcda82 PyObject_Call + 130
22 python 0x00000001071eb89b t_bootstrap + 123
23 python 0x0000000107172937 pythread_wrapper + 39
24 libsystem_pthread.dylib 0x00007fff20332950 _pthread_start + 224
25 libsystem_pthread.dylib 0x00007fff2032e47b thread_start + 15
)
2021-03-14 17:01:37.482 python[38483:2301820] WARNING: nextEventMatchingMask should only be called from the Main Thread! This will throw an exception in the future.
tick
tick
tick
tick
tick
tick
tick
tick
tick
tick
tick
^C^CException ignored in: <module 'threading' from '/usr/local/anaconda3/lib/python3.7/threading.py'>
Traceback (most recent call last):
File "/usr/local/anaconda3/lib/python3.7/threading.py", line 1308, in _shutdown
lock.acquire()
KeyboardInterrupt
I really want my renderer to be in a separate thread.
So much cleaner!
Can it be done?
This doesn't answer the question, but it solves the problem:
import cv2
import numpy as np
from time import sleep
import threading
import time
bitmap = np.zeros((512,512,3),np.uint8)
def update_bitmap():
for i in range(512):
bitmap[i,i,:] = 128
sleep(1/32)
def main():
threading.Thread(target=update_bitmap).start()
hz = 30
delta_t = 1 / hz
t = time.time()
try:
while True:
sleep(0.001)
if time.time() > t+delta_t:
t += delta_t
cv2.imshow("Color Image", bitmap)
cv2.waitKey(1)
except KeyboardInterrupt:
cv2.destroyAllWindows()
exit(0)
main()

tkinter python script crashes when opening from mac command line

So I am trying to make a small GUI in python (version 3.8.3) using tkinter. I wrote the whole script in the PyCharm IDE and I was able to run it without issue and still can. However trying to run it from the command line results in a crash and I get the following output:
2020-07-25 22:22:57.888 Python[69416:1640503] -[SDLApplication
_setup:]: unrecognized selector sent to instance 0x7ffcdd2ad590 2020-07-25 22:22:57.890 Python[69416:1640503] *** Terminating app due
to uncaught exception 'NSInvalidArgumentException', reason:
'-[SDLApplication _setup:]: unrecognized selector sent to instance
0x7ffcdd2ad590'
*** First throw call stack: ( 0 CoreFoundation 0x00007fff378a7797 __exceptionPreprocess + 250 1 libobjc.A.dylib
0x00007fff7073aa9e objc_exception_throw + 48 2 CoreFoundation
0x00007fff379269a6 -[NSObject(NSObject) __retain_OA] + 0 3
CoreFoundation 0x00007fff3780bf30
forwarding + 1427 4 CoreFoundation 0x00007fff3780b908 _CF_forwarding_prep_0 + 120 5 libtk8.6.dylib
0x000000010fd051f2 TkpInit + 408 6 libtk8.6.dylib
0x000000010fc74aac Initialize + 2454 7
_tkinter.cpython-38-darwin.so 0x000000010fac8714 Tcl_AppInit + 84 8 _tkinter.cpython-38-darwin.so 0x000000010fac8032
_tkinter_create + 1362 9 Python 0x000000010f790e8f cfunction_vectorcall_FASTCALL + 175 10 Python
0x000000010f822c4c call_function + 444 11 Python
0x000000010f81faae _PyEval_EvalFrameDefault + 25678 12 Python
0x000000010f823a94 _PyEval_EvalCodeWithName + 2804 13 Python
0x000000010f75504e _PyFunction_Vectorcall + 270 14 Python
0x000000010f7543b7 _PyObject_FastCallDict + 247 15 Python
0x000000010f7559bf _PyObject_Call_Prepend + 143 16 Python
0x000000010f7abb21 slot_tp_init + 145 17 Python
0x000000010f7a6ff9 type_call + 297 18 Python
0x000000010f754565 _PyObject_MakeTpCall + 373 19 Python
0x000000010f822ca5 call_function + 533 20 Python
0x000000010f81faae _PyEval_EvalFrameDefault + 25678 21 Python
0x000000010f823a94 _PyEval_EvalCodeWithName + 2804 22 Python
0x000000010f819584 PyEval_EvalCode + 100 23 Python
0x000000010f8690f0 PyRun_FileExFlags + 336 24 Python
0x000000010f8687e0 PyRun_SimpleFileExFlags + 864 25 Python
0x000000010f885c6f Py_RunMain + 2159 26 Python
0x000000010f885faf pymain_main + 223 27 Python
0x000000010f8861ab Py_BytesMain + 43 28 libdyld.dylib
0x00007fff718d9cc9 start + 1 29 ???
0x0000000000000002 0x0 + 2 ) libc++abi.dylib: terminating with
uncaught exception of type NSException Abort trap: 6
The script ran perfectly when being called from the command line on a Windows computer but for whatever reason I cant get it to run from the command line on my mac, but yet it works in the IDE...
I'll include the code below. I would appreciate any help I can get and if you need more information just ask
import tkinter as tk
import youtube_dl
from moviepy.editor import *
import shutil
BG = "#191970"
# midnight blue #191970
main_window = tk.Tk()
main_window.title("Video Scraper")
main_window.geometry("850x690")
main_window.config(bg=BG)
frame = tk.Frame(main_window, bg="white")
frame.place(relx="0.1", rely="0.05", relwidth=0.8, relheight=0.9)
title = tk.Label(frame, text="Video Scraper")
title.config(font=("Verdana", 44))
title.pack(fill="x")
label = tk.Label(frame, text="Please enter the url(s) of the videos you would like to download so that there's "
"only 1 link per line\nAlso note that there can't be any spaces at the beginning or "
"end of the line", bg="pink")
label.pack()
text_area = tk.Text(frame, bg="#BBFFFF")
text_area.pack(fill="y", expand=True)
frame2 = tk.Frame(main_window, bg=BG)
frame2.place(relx="0.2", rely="0.95", relwidth=0.6, relheight=0.05)
var = tk.IntVar()
var.set(2)
def set_var1():
var.set(1)
def set_var2():
var.set(2)
audio_only = tk.Radiobutton(frame2, text=".mp3", variable="var", value=1, command=set_var1)
vid_and_aud = tk.Radiobutton(frame2, text=".mp4", variable="var", value=2, command=set_var2)
audio_only.pack(side="left")
vid_and_aud.pack(side="left")
def get_text():
text = text_area.get("1.0", "end-1c")
broken_text = text.split("\n")
links = []
for item in broken_text:
link = [item]
links.append(link)
audio_only.config(state="disabled")
vid_and_aud.config(state="disabled")
download_btn.config(state="disabled")
download_videos(links, var.get())
def download_videos(url_links, download_format):
counter = 0
vid_info = []
print(url_links)
print(download_format)
# Get video data for each link and put it into a list
with youtube_dl.YoutubeDL() as ydl:
for link in url_links:
meta = ydl.extract_info(link[0], download=False)
vid_info.append(meta)
ydl_opts = {
'format': 'mp4',
'outtmpl': '%(title)s.mp4',
'quiet': True,
'noplaylist': True
}
while counter < len(url_links):
filename = vid_info[counter]['title'] + ".mp4"
print("\n\nDownloading --> " + filename)
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
ydl.download(url_links[counter])
print("Video Successfully Downloaded!")
if download_format == 1:
print("Converting Video(.mp4) to Audio(.mp3)...")
video = VideoFileClip(filename)
audio_file = filename.replace("mp4", "mp3")
video.audio.write_audiofile(audio_file, logger=None)
video.close()
print("Video Successfully Converted!")
print("Cleaning up...")
# remove video file
print("--> Removing Video(.mp4) File")
os.remove(filename)
# move audio
print("--> Moving Audio to Audio(mp3) Folder")
shutil.move(audio_file, "Audio(mp3)")
print("--> Done")
print("Completed " + str(counter + 1) + "/" + str(len(url_links)))
else:
shutil.move(filename, "Video(mp4)")
print("Completed " + str(counter + 1) + "/" + str(len(url_links)))
counter += 1
download_btn = tk.Button(frame2, text="Download!", command=get_text)
download_btn.pack(side="right")
main_window.mainloop()

opencv throwing an error in macOS - Mojave

macOS - mojave
,
opencv-python = 4.1.0.25
, python = 3.5.6
When I try to open the webcam to capture the image through opencv in macOS-Mojave, I am constantly getting this error.
But running the same script in ubuntu and windows, there is no error I encounter.
Here is my test code
'''
detector = cv2.CascadeClassifier(cascade)
window_name = "Capture"
cv2.namedWindow(window_name, cv2.WND_PROP_FULLSCREEN)
cv2.setWindowProperty(window_name, cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN)
vs = cv2.VideoCapture(0)
vs.set(3, 1280)
vs.set(4, 720)
time.sleep(2.0)
total = 0
msg = "Enter your Name"
title = "Siamese Network Training Module"
fieldNames = ["Name"]
fieldValues = []
**fieldValues = multenterbox(msg,title, fieldNames)**
folder_names = fieldValues[0]
new_output_folder = output + folder_names
print(new_output_folder)
'''
and this is the error at bold line highlighted above
'''
First throw call stack:
(
0 CoreFoundation 0x00007fff4a599a7d __exceptionPreprocess + 256
1 libobjc.A.dylib 0x00007fff74c6aa17 objc_exception_throw + 48
2 CoreFoundation 0x00007fff4a613886 -[NSObject(NSObject) __retain_OA] + 0
3 CoreFoundation 0x00007fff4a53b8ef ___forwarding___ + 1485
4 CoreFoundation 0x00007fff4a53b298 _CF_forwarding_prep_0 + 120
5 libtk8.6.dylib 0x000000012717d31d TkpInit + 413
6 libtk8.6.dylib 0x00000001270d517e Initialize + 2622
7 _tkinter.cpython-35m-darwin.so 0x00000001171799f6 _tkinter_create + 1174
8 python 0x000000010b84455e PyCFunction_Call + 62
9 python 0x000000010b90bec7 PyEval_EvalFrameEx + 18583
.
.
.
.
42 python 0x000000010b957eee PyRun_SimpleFileExFlags + 382
43 python 0x000000010b97cd86 Py_Main + 3622
44 python 0x000000010b7be861 main + 497
45 libdyld.dylib 0x00007fff764383d5 start + 1
46 ??? 0x0000000000000002 0x0 + 2
)
libc++abi.dylib: terminating with uncaught exception of type NSException
Abort trap: 6
'''
Please let me know what can be done. Thanks

Improve performance of python script to filter unpaired reads in a pair of FASTQ files

I have some paired end sequencing data. However, the files are not properly paired. There are unpaired reads which need to be removed to make the read pair files consistent. Though there are solutions like using Trimmomatic. I want suggestions as to how I can improve the performance of my own script. The current version processes at about 10K records per second.
import sys
class FastqRecord(object):
def __init__(self, block, unique_col=1, unique_type=int, sep=' '):
self.block = "".join(block[:])
self.header = block[0][1:].rstrip('\n')
self.uid = unique_type(self.header.split(sep)[unique_col])
def __eq__(self, other):
return self.uid == other.uid
def __gt__(self, other):
return self.uid > other.uid
class Fastq(object):
def __init__(self, filename):
self.file = filename
self.handle = open(self.file)
self.count = 0
self.record = self.next()
def __iter__(self):
return self
def next(self):
return FastqRecord([self.handle.next(), self.handle.next(), self.handle.next(), self.handle.next()])
def update(self):
try:
self.record = self.next()
self.count+=1
except StopIteration:
self.record = False
self.handle.close()
def fn_from_path(path):
return path.split('/')[-1].split('.')[0]
def flush_handle(obj, handle):
handle.write(obj.record.block)
for x in obj:
handle.write(x.block)
return True
#profile
def main():
file1, file2, out_dir = (sys.argv[1], sys.argv[2], sys.argv[3].rstrip('/'))
#file1, file2, out_dir = ('./test/SRR2130002_1.fastq', './test/SRR2130002_2.fastq', '.')
out_handles = {
'p1': open('%s/%s.fastq' % (out_dir, fn_from_path(file1)), 'w'),
'p2': open('%s/%s.fastq' % (out_dir, fn_from_path(file2)), 'w'),
's': open('%s/%s_unpaired.fastq' % (out_dir, fn_from_path(file1).split('_')[0]), 'w')
}
f1, f2 = Fastq(file1), Fastq(file2)
while True:
print "\r%9d %9d %9d" % (f1.count, f2.count, f1.count-f2.count),
sys.stdout.flush()
if f1.record is False or f2.record is False:
if f1.record is False:
flush_handle(f2, out_handles['s'])
else:
flush_handle(f1, out_handles['s'])
break
if f1.record == f2.record:
out_handles['p1'].write(f1.record.block)
out_handles['p2'].write(f2.record.block)
f1.update()
f2.update()
elif f1.record > f2.record:
out_handles['s'].write(f2.record.block)
f2.update()
else:
out_handles['s'].write(f1.record.block)
f1.update()
for i in out_handles:
out_handles[i].close()
print "\n Done!"
if __name__ == "__main__":
main()
This is how my FASTQ files look:
head SRR2130002_1.fasta
#SRR2130002.1 1 length=39
CCCTAACCCTAACCCTAACCCTAACCCAAAGACAGGCAA
+SRR2130002.1 1 length=39
AAAAA7F<<FF<F.<FFFFF77F.))))FA.))))))7A
#SRR2130002.2 2 length=39
TTAGGGTTAGGGTTAGGGTTAGGGTTAGGGTATCTCGTA
+SRR2130002.2 2 length=39
<AAAAAA.7FFFFFFFFFF.<...).A.F7F7)A.FAA<
#SRR2130002.3 3 length=39
CTACCCCTAACCCTAACCCTAACAAAACCCCAAAACAAA
Thanks
UPDATE:
I ran a line profiler (kernprof) on the main() function. And got following stats:
Timer unit: 1e-06 s
Total time: 0.808629 s
File: paired_extractor.py
Function: main at line 46
Line # Hits Time Per Hit % Time Line Contents
==============================================================
46 #profile
47 def main():
48 1 4 4.0 0.0 file1, file2, out_dir = (sys.argv[1], sys.argv[2], sys.argv[3].rstrip('/'))
49 #file1, file2, out_dir = ('./test/SRR2130002_1.fastq', './test/SRR2130002_2.fastq', '.')
50 1 1 1.0 0.0 out_handles = {
51 1 4830 4830.0 0.6 'p1': open('%s/%s.fastq' % (out_dir, fn_from_path(file1)), 'w'),
52 1 4118 4118.0 0.5 'p2': open('%s/%s.fastq' % (out_dir, fn_from_path(file2)), 'w'),
53 1 1283 1283.0 0.2 's': open('%s/%s_unpaired.fastq' % (out_dir, fn_from_path(file1).split('_')[0]), 'w')
54 }
55 1 2945 2945.0 0.4 f1, f2 = Fastq(file1), Fastq(file2)
56 25001 23354 0.9 2.9 while True:
57 25001 105393 4.2 13.0 print "\r%9d %9d %9d" % (f1.count, f2.count, f1.count-f2.count),
58 25001 68264 2.7 8.4 sys.stdout.flush()
59 25001 29858 1.2 3.7 if f1.record is False or f2.record is False:
60 1 1 1.0 0.0 if f1.record is False:
61 1 9578 9578.0 1.2 flush_handle(f2, out_handles['s'])
62 else:
63 flush_handle(f1, out_handles['s'])
64 1 1 1.0 0.0 break
65 25000 47062 1.9 5.8 if f1.record == f2.record:
66 23593 41546 1.8 5.1 out_handles['p1'].write(f1.record.block)
67 23593 34040 1.4 4.2 out_handles['p2'].write(f2.record.block)
68 23593 216319 9.2 26.8 f1.update()
69 23593 196077 8.3 24.2 f2.update()
70 1407 2340 1.7 0.3 elif f1.record > f2.record:
71 out_handles['s'].write(f2.record.block)
72 f2.update()
73 else:
74 1407 2341 1.7 0.3 out_handles['s'].write(f1.record.block)
75 1407 13231 9.4 1.6 f1.update()
76 4 9 2.2 0.0 for i in out_handles:
77 3 6023 2007.7 0.7 out_handles[i].close()
78 1 11 11.0 0.0 print "\n Done!"
Let me caveat my response with the fact that it's typically hard/impossible to optimize code without doing due-diligence with a profiler (which is what I'm about to do, since I don't have your fasta files). You also seem to have some good habits with strings (like using join). It looks like you don't need
print "\r%9d %9d %9d" % (f1.count, f2.count, f1.count-f2.count),
sys.stdout.flush()
as this is just outputting stats to console. You'll get a little speedup by removing that...but nothing major jumped out at me.
Just as a matter of personal opinion, if you want general processing speed, I'd suggest using C/C++. They don't have the overhead of the interpreter. The Python interpreter is an amazing tool, but it wasn't built for speed. It was built for correctness/security/ease-of-use. Specifically for string manipulations, Perl is king from what I hear, although admittedly I've only written 10s of lines of Perl in my life.

Python Library for LSM303D 6-Axis Accelerometer / Compass V2.0: unexpected results

I recently acquired a Grove Sensor board for the LSM303D 6-Axis Accelerometer / Compass. Unfortunately, it transpired that the device did not yet have a Python library (only a C++ library and Arduino .ino file).
I am attempting to use it with the GrovePi+ board with Raspberry Pi, and would like to use it with Python code, so I made an attempt to transcribe the C++ Code into Python. However, I am getting odd output.
Additional information relating to the device can be found on the Pololu Datasheet
sudo i2cdetect -y 1 shows the device can be found at address 0x1E
0 1 2 3 4 5 6 7 8 9 a b c d e f
00: -- 04 -- -- -- -- -- -- -- -- -- -- --
10: -- -- -- -- -- -- -- -- -- -- -- -- -- -- 1e --
20: -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
30: -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
40: -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
50: -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
60: -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
70: -- -- -- -- -- -- -- --
Here is the Python Code:
#!/usr/bin.env python
import smbus
import time
import math
import RPi.GPIO as GPIO
import struct
rev = GPIO.RPI_REVISION
if rev == 2 or rev == 3:
bus = smbus.SMBus(1)
else:
bus = smbus.SMBus(0)
### LSM303 Address ###
LSM303D_ADDR = 0x1E # Assuming SA0 grounded
### LSM303 Register definitions ###
TEMP_OUT_L = 0x05
TEMP_OUT_H = 0x06
STATUS_REG_M = 0x07
OUT_X_L_M = 0x08
OUT_X_H_M = 0x09
OUT_Y_L_M = 0x0A
OUT_Y_H_M = 0x0B
OUT_Z_L_M = 0x0C
OUT_Z_H_M = 0x0D
WHO_AM_I = 0x0F
INT_CTRL_M = 0x12
INT_SRC_M = 0x13
INT_THS_L_M = 0x14
INT_THS_H_M = 0x15
OFFSET_X_L_M = 0x16
OFFSET_X_H_M = 0x17
OFFSET_Y_L_M = 0x18
OFFSET_Y_H_M = 0x19
OFFSET_Z_L_M = 0x1A
OFFSET_Z_H_M = 0x1B
REFERENCE_X = 0x1C
REFERENCE_Y = 0x1D
REFERENCE_Z = 0x1E
CTRL_REG0 = 0x1F
CTRL_REG1 = 0x20
CTRL_REG2 = 0x21
CTRL_REG3 = 0x22
CTRL_REG4 = 0x23
CTRL_REG5 = 0x24
CTRL_REG6 = 0x25
CTRL_REG7 = 0x26
STATUS_REG_A = 0x27
OUT_X_L_A = 0x28
OUT_X_H_A = 0x29
OUT_Y_L_A = 0x2A
OUT_Y_H_A = 0x2B
OUT_Z_L_A = 0x2C
OUT_Z_H_A = 0x2D
FIFO_CTRL = 0x2E
FIFO_SRC = 0x2F
IG_CFG1 = 0x30
IG_SRC1 = 0x31
IG_THS1 = 0x32
IG_DUR1 = 0x33
IG_CFG2 = 0x34
IG_SRC2 = 0x35
IG_THS2 = 0x36
IG_DUR2 = 0x37
CLICK_CFG = 0x38
CLICK_SRC = 0x39
CLICK_THS = 0x3A
TIME_LIMIT = 0x3B
TIME_LATENCY = 0x3C
TIME_WINDOW = 0x3D
ACT_THS = 0x3E
ACT_DUR = 0x3F
### Mag scales ###
MAG_SCALE_2 = 0x00 # full-scale is +/- 2 Gauss
MAG_SCALE_4 = 0x20 # +/- 4 Guass
MAG_SCALE_8 = 0x40 # +/- 8 Guass
MAG_SCALE_12 = 0x60 # +/- 12 Guass
ACCEL_SCALE = 2 # +/- 2g
X = 0
Y = 1
Z = 2
def twos_comp(val, bits):
# Calculate the 2s complement of int:val #
if(val&(1<<(bits-1)) != 0):
val = val - (1<<bits)
return val
# return val if val < 32768 else val - 65536
class accelcomp:
mag = [0,0,0]
accel = [0,0,0]
tiltcomp = [0,0,0]
heading=0
headingDegrees=0
tiltHeading=0
tiltHeadingDegrees=0
def __init__(self):
whoami = bus.read_byte_data(LSM303D_ADDR, WHO_AM_I)
if(whoami == 0x49):
bus.write_byte_data(LSM303D_ADDR, CTRL_REG1, 0x57) # 0x57 = ODR=50hz, all accel axes on ## maybe 0x27 is Low Res?
bus.write_byte_data(LSM303D_ADDR, CTRL_REG2, (3<<6)|(0<<3)) # set full scale +/- 2g
bus.write_byte_data(LSM303D_ADDR, CTRL_REG3, 0x00) # no interrupt
bus.write_byte_data(LSM303D_ADDR, CTRL_REG4, 0x00) # no interrupt
bus.write_byte_data(LSM303D_ADDR, CTRL_REG5, (4<<2)) # 0x10 = mag 50Hz output rate
bus.write_byte_data(LSM303D_ADDR, CTRL_REG6, MAG_SCALE_2) # Magnetic Scale +/1 1.3 Guass
bus.write_byte_data(LSM303D_ADDR, CTRL_REG7, 0x00) # 0x00 continuous conversion mode
def getMag(self):
self.mag[X] = twos_comp(bus.read_byte_data(LSM303D_ADDR, OUT_X_H_M) << 8 |
bus.read_byte_data(LSM303D_ADDR, OUT_X_L_M), 16)
self.mag[Y] = twos_comp(bus.read_byte_data(LSM303D_ADDR, OUT_Y_H_M) << 8 |
bus.read_byte_data(LSM303D_ADDR, OUT_Y_L_M), 16)
self.mag[Z] = twos_comp(bus.read_byte_data(LSM303D_ADDR, OUT_Z_H_M) << 8 |
bus.read_byte_data(LSM303D_ADDR, OUT_Z_L_M), 16)
def getAccel(self):
accel = [0,0,0]
accel[X] = twos_comp(bus.read_byte_data(LSM303D_ADDR, OUT_X_H_A) << 8 |
bus.read_byte_data(LSM303D_ADDR, OUT_X_L_A), 16)
accel[Y] = twos_comp(bus.read_byte_data(LSM303D_ADDR, OUT_Y_H_A) << 8 |
bus.read_byte_data(LSM303D_ADDR, OUT_Y_L_A), 16)
accel[Z] = twos_comp(bus.read_byte_data(LSM303D_ADDR, OUT_Z_H_A) << 8 |
bus.read_byte_data(LSM303D_ADDR, OUT_Z_L_A), 16)
for i in range(X, Z+1):
self.accel[i] = accel[i] / math.pow(2, 15) * ACCEL_SCALE
def getHeading(self):
self.heading = math.atan2(self.mag[X], self.mag[Y])
if self.heading < 0:
self.heading += 2*math.pi
if self.heading > 2*math.pi:
self.heading -= 2*math.pi
self.headingDegrees = round(math.degrees(self.heading),2)
def getTiltHeading(self):
truncate = [0,0,0]
for i in range(X, Z+1):
truncate[i] = math.copysign(min(math.fabs(self.accel[i]), 1.0), self.accel[i])
try:
pitch = math.asin(-1*truncate[X])
roll = math.asin(truncate[Y]/math.cos(pitch)) if abs(math.cos(pitch)) >= abs(truncate[Y]) else 0
# set roll to zero if pitch approaches -1 or 1
self.tiltcomp[X] = self.mag[X] * math.cos(pitch) + self.mag[Z] * math.sin(pitch)
self.tiltcomp[Y] = self.mag[X] * math.sin(roll) * math.sin(pitch) + \
self.mag[Y] * math.cos(roll) - self.mag[Z] * math.sin(roll) * math.cos(pitch)
self.tiltcomp[Z] = self.mag[X] * math.cos(roll) * math.sin(pitch) + \
self.mag[Y] * math.sin(roll) + \
self.mag[Z] * math.cos(roll) * math.cos(pitch)
self.tiltHeading = math.atan2(self.tiltcomp[Y], self.tiltcomp[X])
if self.tiltHeading < 0:
self.tiltHeading += 2*math.pi
if self.tiltHeading > 2*math.pi:
self.heading -= 2*math.pi
self.tiltHeadingDegrees = round(math.degrees(self.tiltHeading),2)
except Exception:
print "AccelX {}, AccelY {}".format(self.accel[X], self.accel[Y])
print "TruncX {}, TruncY {}".format(truncate[X], truncate[Y])
print "Pitch {}, cos(pitch) {}, Bool(cos(pitch)) {}".format(pitch, math.cos(pitch), bool(math.cos(pitch)))
def isMagReady(self):
temp = bus.read_byte_data(LSM303D_ADDR, STATUS_REG_M) & 0x03
return temp
def update(self):
self.getAccel()
self.getMag()
self.getHeading()
self.getTiltHeading()
if __name__ == '__main__':
from time import sleep
lsm = accelcomp()
print ' Accelerometer || Compass || Heading'
print ' x | y | z || x | y | z || calc | tilt corrected |'
while True:
lsm.getAccel()
while not bool(lsm.isMagReady()):
print "Not Ready!"
sleep(.01)
lsm.getMag()
lsm.getHeading()
lsm.getTiltHeading()
print '{0:.3f} |{1:.3f} |{2:.3f} ||{3:6} |{4:6} |{5:6} || {6:6} deg | {7:6} deg |'.format(
lsm.accel[X], lsm.accel[Y], lsm.accel[Z], lsm.mag[X], lsm.mag[Y], lsm.mag[Z], lsm.headingDegrees, lsm.tiltHeadingDegrees)
sleep(.5)
Some sample output:
Accelerometer || Compass || Heading
x | y | z || x | y | z || calc | tilt corrected |
0.022 |-0.012 |1.056 || 876 | -1119 | 5788 || 141.94 deg | 305.58 deg |
0.027 |-0.011 |1.052 || 919 | -1128 | 5826 || 140.83 deg | 305.62 deg |
0.026 |-0.010 |1.053 || 903 | -1108 | 5777 || 140.82 deg | 305.61 deg |
0.024 |-0.009 |1.054 || 914 | -1140 | 5790 || 141.28 deg | 305.57 deg |
0.024 |-0.010 |1.056 || 895 | -1122 | 5801 || 141.42 deg | 305.42 deg |
0.024 |-0.010 |1.055 || 905 | -1088 | 5747 || 140.25 deg | 306.69 deg |
0.022 |-0.008 |1.057 || 900 | -1106 | 5748 || 140.86 deg | 306.15 deg |
One point that is obvious from the output displayed is that the acceleration in the Z direction is, sitting on the table, greater than 1g. It's close enough to be considered ok, although when tilted, both X and Y also go over 1g, which throws Math Errors with the asin function in getTiltHeading(), hence the truncation.
However, the real issue is with errors in the compass over a full rotation. I aligned the device north relative to the compass on my phone, and it gave a close value to 360 / 0 degrees. However, rotating it clockwise around the circle introduced errors that climbed up to about 30 degrees off the expected value around 255 degrees (near magnetic West).
Tabulated Experiment:
Angle Trial 1 Trial 2 Trial 3 Trial 4 AVERAGE ERROR
0 356.53 355.18 355.74 357.13 356.145 356.145
0 -3.47 -4.82 -4.26 -2.87 -3.855 -3.855
15 11.26 10.65 11.57 11.02 11.125 -3.875
30 30.29 30.82 30.74 30.29 30.535 0.535
45 40.24 40.32 39.95 39.93 40.11 -4.89
60 55.84 55.51 55.53 54.89 55.4425 -4.5575
75 67.23 67.42 68.46 68.23 67.835 -7.165
90 83.65 83.8 83.96 83.55 83.74 -6.26
105 101.11 100.91 100.13 100.66 100.7025 -4.2975
120 117.15 117.13 116.59 115.79 116.665 -3.335
135 125.92 125.56 125.19 126.14 125.702 -9.2975
150 142.74 142.44 142.94 142.56 142.67 -7.33
165 157.22 156.77 157.53 157.07 157.1475 -7.8525
180 171.97 171.61 172.17 172.26 172.0025 -7.9975
195 183.81 183.47 182.99 183.39 183.415 -11.585
210 195.59 196.6 197.2 197.58 196.7425 -13.2575
225 210.07 210.59 209.47 210.21 210.085 -14.915
240 218.97 219.3 219.53 219.48 219.32 -20.68
255 226.13 226.27 226.61 226.18 226.2975 -28.7025
270 242.63 241.64 242.33 242.32 242.23 -27.77
285 259.89 258.83 258.92 259 259.16 -25.84
300 283.8 284.51 283.51 282.87 283.6725 -16.3275
315 294.05 295.47 295.23 295.57 295.08 -19.92
330 320.09 319.28 318.64 318.16 319.0425 -10.9575
345 342.39 342.87 342.34 343.13 342.6825 -2.3175
360 8.85 8.48 8.93 9.1 8.84 8.84
Graph of errors:
Is there some error I've made in the library, or might I be dealing with a faulty device?
(The dotted trendline is a 6th-order polynomial, to which I doubt indicates any actual relation. After all, I would expect this function to be cyclic.)
Just a little late to the party ;p
In your getTiltHeading function, this line:
self.heading -= 2*math.pi
might need to be:
self.tiltHeading -= 2*math.pi

Categories

Resources