I'm trying to add a multicast network trigger for a camera set up. Currently I am using a conf. file called multicast-trigger.conf
[multicast-trigger]
address = 224.1.1.1
port = 600
payload = 0x05AA9544
and a python script called app_ext_multicast_trigger.py which takes the information from the conf file to send a multicast network trigger.
import sys, os, logging, json, ConfigParser, socket
sys.path.append('/home/root/ss-web')
from camconstants import *
class AppExt(object):
pkg_name = "Multicast network trigger "
pkg_version = "v1.0"
mt_config = None
# =============================================================================
# Package and URL registration logic
# =============================================================================
def __init__(self, _app, _cam, _ci, register_url_callback):
"""
Register new URLs with camera's webserver.
"""
urls = [
( '/trigger2', self.trigger2, "Sends a multicast network trigger packet to trigger all the cameras on the same local network, including trigging this camera." ),
( '/get_multicast_configuration', self.get_multicast_configuration, "Returns a JSON encoded dictionary of the camera's /etc/multicast-trigger.conf file contents." ),
]
register_url_callback(self.pkg_name, self.pkg_version, urls)
self.mt_config = self._mt_read_multicast_config_file('/etc/multicast-trigger.conf')
logging.debug("Multicast trigger URL added: %s" % repr(self.mt_config))
# =============================================================================
# Multicast trigger helper methods
# =============================================================================
def _mt_read_multicast_config_file(self, fn):
"""
Returns dictionary with multicast trigger configuration information read from file fn.
"""
if not os.path.exists(fn):
logging.error("Missing file: %s" % fn)
return None
config = {}
try:
config_parser = ConfigParser.SafeConfigParser()
config_parser.read(fn)
c = config_parser.items('multicast-trigger')
for (key, value) in c:
if key == 'address':
config[key] = value
elif key in ('port', 'payload'):
config[key] = int(value, 0)
except Exception, e:
logging.error("Bad file format: %s" % fn)
logging.error("Ignoring multicast-trigger parameters due to exception - %s" % str(e))
return None
return config
# =============================================================================
# Exposed URLs - URL matches method name
# =============================================================================
def trigger2(self):
"""
Sends a multicast network packet to trigger all cameras on the same local network.
"""
if self.mt_config == None:
logging.error("Missing file: %s" % fn)
ret = CAMAPI_STATUS_INVALID_PARAMETER
else:
try:
logging.debug("Triggering cameras by sending multicast packet: address %s port %d with payload 0x%x" % (self.mt_config['address'], self.mt_config['port'], self.mt_config['payload']))
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 32)
data=""
v = self.mt_config['payload']
for i in range(4):
data = chr(v & 0xFF) + data
v = v >> 8
sock.sendto(data, (self.mt_config['address'], self.mt_config['port']))
ret = CAMAPI_STATUS_OKAY
except Exception, e:
logging.error("Multi-camera trigger error due to exception - %s" % str(e))
ret = CAMAPI_STATUS_INVALID_PARAMETER
return json.dumps(ret)
def get_multicast_configuration(self):
"""
Returns a JSON encoded dictionary of the camera's /etc/multicast-trigger.conf file.
"""
logging.debug("Returning multicast trigger configuration: %s" % repr(self.mt_config))
return json.dumps(self.mt_config)
This works well and when I go to the ip address of the camera /trigger2 it will send a packet to trigger the other camera on the same network. Now I also have a different piece of equipment which will also trigger the camera via a python script called app_ext_b1 and when run from the device will trigger the camera and save to the device.
import sys, os, logging, json, time, threading, flask, glob, datetime, subprocess, math
from urllib import unquote
from camconstants import *
import tarfile
import sys, os, time, json, datetime, math
class CameraException(Exception):
def __init__(self, code, message):
self.Code = code
self.Message = message
super(CameraException, self).__init__(message)
def AsObject(self):
return { "Code" : self.Code, "Message" : self.Message }
class ExecutionContext:
def __init__(self, id, path):
self.Id = id
self.Error = None
self.path = path
if not self.path.endswith("/"):
self.path = "/" + self.path
self.Key = None
self.Logs = []
self.Results = {}
self._from = time.time()
self.add_log("Id: " + self.Id)
def get_path(self, fname):
return self.path + fname
def execute(self, key, method):
self.Key = key
self.add_log("Executing " + key)
try:
method(self)
except CameraException as camException:
self.add_log("Failed task " + key)
self._add_error(key, camException)
return False
except Exception as error:
self.add_log("Failed task " + key)
self._add_error(key, CameraException(-1, str(error)))
return False
self.add_log("Finished " + key)
return True
def _add_error(self, key, error):
self.Error = error
self.add_log("Error: " + str(error) )
def add_local(self, key, result):
if not self.Key in self.Results:
self.Results[self.Key] = {}
local_result = self.Results[self.Key]
local_result[key] = result
def add_log(self, message):
delta = str(int(1000*(time.time() - self._from))).rjust(6)
fullmsg = str(delta) + ": " + str(self.Key) + " " + message
self.Logs.append(fullmsg)
def add_state(self, save_state, camStatus):
state = {
"save_state": save_state,
"time": int(time.time() * 1000),
"camerastatus": camStatus,
}
stateStr = json.dumps(state)
self.add_log("Setting state to: " + stateStr)
progress_file = open(self.get_path(self.Id + ".state"), "w")
progress_file.write(stateStr)
progress_file.close()
def get_state(self):
try:
stateJson = open(self.get_path(self.Id + ".state"), "r").read()
self.add_log("State: " + stateJson)
state = json.loads(stateJson)
return state
except:
return None
def save_logs(self):
try:
self.Logs.append("Results:\n" + json.dumps(self.Results))
progress_file = open(self.get_path(self.Id + ".log"), "w")
progress_file.write("\n".join(self.Logs))
progress_file.close()
except:
pass
class TriggerInfo:
def __init__(self, triggerTime, releaseTime, fps):
self.TriggerTime = triggerTime
if releaseTime <=0:
releaseTime = triggerTime + releaseTime
self.ReleaseTime = releaseTime
self.Fps = fps
def getFrameForDeltaReleaseMs(self, deltaReleaseMs):
return self.convertMsToFrames(((self.ReleaseTime + float(deltaReleaseMs) / 1000.0) - self.TriggerTime)* 1000)
def convertMsToFrames(self, ms):
return int(self.Fps * float(ms)/ 1000.0)
class FrameSaver:
def __init__(self, cam, ec, params):
self.Cam = cam
self.Tar = None
self.Ec = ec
self.LastImageModified = None
self.id = params["id"]
frameTo = params["frameTo"]
curFrame = params["frameFrom"]
self.Frames = [curFrame]
interval = params["frameInterval"]
curFrame += interval
while curFrame < frameTo:
self.Frames.append(curFrame)
curFrame += interval
self.Frames.append(frameTo)
if not frameTo in self.Frames:
self.Frames.append(frameTo)
def save_frames(self):
for frameNumber in self.Frames:
self._remove_image()
self.Ec.add_log("reviewing frame")
self.Cam.review_frame(1, frameNumber)
self._save_image(frameNumber)
self.Tar.close()
def _remove_image(self):
os.system("rm /home/root/ss-web/static/images/image.jpg")
def _save_image(self, frameNumber):
self.Ec.add_log("save image to tar")
# http://10.11.12.13/static/images/image.jpg
path = "/home/root/ss-web/static/images/image.jpg"
tar = self.create_archive_if_not_exists()
start_time = time.time()
while not os.path.exists(path):
time.sleep(0.050)
if (time.time() - start_time) > 1:
raise CameraException(-1, "Fullball flight: failed to save image for framenumber " + str(frameNumber))
tar.add(path, str(frameNumber) + ".jpg")
def create_archive_if_not_exists(self):
if self.Tar == None:
self.Tar = tarfile.open("/mnt/sdcard/DCIM/" + self.id + ".Frames.tar", "w")
return self.Tar
class CameraCaptureFlow:
def __init__(self, id, camera, options):
self.Camera = camera
self.Options = options
self._state_key = "Unknown"
self.Id = id
self.Fps = 1
self.TriggerInfo = None
self.ReleaseFrame = None
self.SaveFps = None
self.StartFrame = None
self.EndFrame = None
self.PretriggerFillLevel = 100
self.ExecutionContext = ExecutionContext(id, "/mnt/sdcard/DCIM/")
self.ExecutionContext.add_log("Options:\n" + json.dumps(options))
def run(self):
# A trigger flow has already been initiated this id - return - note, it can be None if we are generating the file on top of the old one
if self.ExecutionContext.get_state() != None:
return
self._add_state("Started")
if not self._execute("Info", self.info):
return
if not self._execute("trigger", self.trigger):
self._execute("reinit", self.reinit)
return
if not self._execute("saveselected", self.save_selected):
self._execute("reinit", self.reinit)
return
if not self._execute("remaining ball flight", self.save_remaning_ballflight):
self._execute("reinit", self.reinit)
return
if not self._execute("reinit", self.reinit):
return
self._add_state("Completed")
return
def info(self, context):
self.CurrentConfig = self.Camera.get_current_settings()
self.PretriggerFillLevel = self.Camera.get_pretrigger_fill_level()
self.CamInfo = self.Camera.get_storage_info()
if not "available_space" in self.CamInfo:
raise CameraException(3, "StorageUnavaible")
if self.CamInfo["available_space"] < 50000000:
raise CameraException(4, "StorageInsufficient")
self.Fps = self.CurrentConfig["frame_rate"]
def TriggerCamera(self, context):
if hasattr(self.Camera, "trigger_hw"):
context.add_log("using hardware trigger")
return self.Camera.trigger_hw(None)
return self.Camera.trigger(None)
def wait_for_buffer_ready(self, context, releaseOffset, postCaptureMs):
delta_ms_release = int(1000*(time.time() - releaseOffset))
waitTime = postCaptureMs - delta_ms_release
context.add_log("Time since release : " + str(delta_ms_release) + "ms" )
context.add_log("Post capture time : " + str(postCaptureMs) + "ms" )
if waitTime > 0:
context.add_log("Time since release is less than required for post capture duration")
context.add_log("waiting " +str(waitTime + 100) + "ms to fill up buffers")
time.sleep((waitTime)/1000.0 + 0.1)
def trigger(self, context):
releaseOffset = self.Options["releaseOffsetTime"]
self.wait_for_buffer_ready(context, releaseOffset, self.Options["postCaptureMs"])
context.add_log("Triggering camera")
before = time.time()
triggerResult = self.TriggerCamera(context)
after = time.time()
triggerTime = (before + after) / 2
self.TriggerInfo = TriggerInfo(triggerTime, releaseOffset, self.Fps)
context.add_local("triggerTime", triggerTime)
context.add_local("relaseTime", self.TriggerInfo.ReleaseTime)
if (triggerResult != CAMAPI_STATUS_OKAY):
self._stop_if_saving()
raise CameraException(6, "TriggerFail")
context.add_log("Waiting for camera to trigger")
self.wait_for_status(lambda status: status == CAMAPI_STATE_TRIGGERED, 1)
context.add_log("Waiting for camera to finish triggering")
self.wait_for_status(lambda status: status != CAMAPI_STATE_TRIGGERED, 2)
context.add_log("Triggering finished")
def _stop_if_saving(self):
camStatus = self.Camera.get_camstatus()["state"]
if camStatus == CAMAPI_STATE_SELECTIVE_SAVING:
self.Camera.save_stop(discard_unsaved=True)
self.wait_for_status(lambda status: status == CAMAPI_STATE_REVIEWING or status == CAMAPI_STATE_RUNNING, 5)
raise CameraException(5, "TriggerFailCameraIsSaving")
def save_selected(self, context):
preCapMs = self.Options["preCaptureMs"]
postCapMs = self.Options["postCaptureMs"]
self.ReleaseFrame = self.TriggerInfo.getFrameForDeltaReleaseMs(0)
self.StartFrame = self.TriggerInfo.getFrameForDeltaReleaseMs(-preCapMs)
self.EndFrame = self.TriggerInfo.getFrameForDeltaReleaseMs(postCapMs)
context.add_log("ReleaseFrame: " + str(self.ReleaseFrame))
context.add_local("release_frame", self.ReleaseFrame)
context.add_local("start_frame", self.StartFrame)
context.add_local("end_frame", self.EndFrame)
self._validateSaveParams(context, self.StartFrame, self.EndFrame)
save_params = {}
save_params['buffer_number'] = 1
save_params['start_frame'] = self.StartFrame
save_params['end_frame'] = self.EndFrame
save_params['filename'] = str(self.Options["id"])
context.add_log("selective_save_params\n" + json.dumps(save_params))
before = time.time()
if self.Camera.selective_save(save_params) != CAMAPI_STATUS_OKAY:
raise CameraException(9, "SelectiveSaveFailed")
context.add_log("Waiting for camera to start saving")
self.wait_for_status(lambda status: status == CAMAPI_STATE_SELECTIVE_SAVING, 1)
context.add_log("Waiting for camera to finish saving")
self.wait_for_status(lambda status: status != CAMAPI_STATE_SELECTIVE_SAVING, 300)
context.add_log("Camera finished saving")
after = time.time()
self.SaveFps = math.ceil((self.EndFrame - self.StartFrame) / (after - before))
context.add_local("save_fps", self.SaveFps)
context.add_log("save fps: " + str(self.SaveFps))
def save_remaning_ballflight(self, context):
context.add_log("Checking options")
frameIntervalMs = self.Options["singleFrameCaptureIntervalMs"]
frameCaptureEndMs = self.Options["singleFrameCaptureEndMs"]
# we wish to combine the existing video with the next frames
if frameIntervalMs == None:
return
frameCaptureStartMs = self.Options["postCaptureMs"] + frameIntervalMs
frameInterval = self.TriggerInfo.convertMsToFrames(frameIntervalMs)
frameFrom = self.TriggerInfo.getFrameForDeltaReleaseMs(frameCaptureStartMs)
frameEnd = self.TriggerInfo.getFrameForDeltaReleaseMs(frameCaptureEndMs)
frameSaverParams = {
"id" : self.Id,
"frameFrom" : frameFrom,
"frameTo" : frameEnd,
"frameInterval" : frameInterval
}
frameSaver = FrameSaver(self.Camera, self.ExecutionContext, frameSaverParams)
frameSaver.save_frames()
def _validateSaveParams(self, context, startFrame, endFrame):
duration = self.CurrentConfig["duration"]
pretrigger = self.CurrentConfig["pretrigger"]
context.add_log("Pretrigger Level " + str(self.PretriggerFillLevel))
preTriggerBufferSeconds = duration * pretrigger / 100.0 * (float(self.PretriggerFillLevel) *1.0001 / 100.0)
context.add_log("preTriggerBufferSeconds " + str(preTriggerBufferSeconds))
postTriggerBuffserSeconds = duration * (1 - pretrigger / 100)
minFrame = -preTriggerBufferSeconds * self.Fps + 10
maxFrame = postTriggerBuffserSeconds * self.Fps - 10
if startFrame < minFrame:
msg = "Startframe: " + str(startFrame) + " is less than minimum frame" + str(minFrame)
context.add_log(msg)
raise CameraException(7, "OutOfBufferStartFrame")
if endFrame > maxFrame:
msg = "Endframe: " + str(endFrame) + " is larger than maximum frame" + str(maxFrame)
context.add_log(msg)
raise CameraException(8, "OutOfBufferEndFrame")
def reinit(self, context):
self.Camera.run(self.CurrentConfig)
def wait_for_status(self, predicate, timeout):
before = time.time()
lastStateUpdate = time.time()
lastState = None
while True:
status = self.Camera.get_camstatus()["state"]
deltaTime = time.time() - lastStateUpdate
if status != lastState or deltaTime > 1:
lastState = status
self._update_state(status)
lastStateUpdate = time.time()
if predicate(status):
return True
if time.time() - before > timeout:
return False
time.sleep(.333)
return False
def _update_state(self, camStatus):
self.ExecutionContext.add_state(self._state_key, camStatus)
def _add_state(self, save_state_key):
self._state_key = save_state_key
try:
camStatus = self.Camera.get_camstatus()["state"]
except:
camStatus = CAMAPI_STATE_UNCONFIGURED
self._update_state(camStatus)
def _execute(self, key, method):
success = self.ExecutionContext.execute(key, method)
if not success:
self._add_state("Failed")
return success
def save_logs(self):
self.ExecutionContext.save_logs()
class AppExt(object):
pkg_name = "B1 application extension"
pkg_version = "v1.8.1"
app = None
cam = None
ci = None
def __init__(self, _app, _cam, _ci, register_url_callback):
self.app = _app
self.cam = _cam
self.ci = _ci
urls = [
(
'/get_extension_b1_version',
self.get_version,
"returns version of B1 extension"
),
(
'/get_time_offset',
self.get_time_offset,
"Gets the camera time offset"
),
(
'/trigger_and_save',
self.trigger_and_save,
"Triggers the camera, and save the selected capture part"
),
(
'/get_trigger_and_save_progress',
self.get_trigger_and_save_progress,
"Returns the progress of the trigger and save process"
),
(
'/ensure_camera_ready',
self.ensure_camera_ready,
"Will check that the camera can save and has enough available space"
)
]
register_url_callback(self.pkg_name, self.pkg_version, urls)
def get_version(self):
return json.dumps(
{
"extensionVersion": AppExt.pkg_version
})
def get_time_offset(self):
timeOffset = time.time()
return json.dumps(
{
"timeOffset": timeOffset
})
def trigger_and_save(self):
request = {}
try:
request["id"] = str(self.get_arg('id'))
request["releaseOffsetTime"] = float(self.get_arg('releaseOffsetTime'))
request["preCaptureMs"] = self.get_int('preCaptureMs')
request["postCaptureMs"] = self.get_int('postCaptureMs')
request["singleFrameCaptureIntervalMs"] = self.get_int('singleFrameCaptureIntervalMs', True)
request["singleFrameCaptureEndMs"] = self.get_int('singleFrameCaptureEndMs', True)
captureFlow = CameraCaptureFlow(request["id"], self.cam, request)
timeStart = time.time()
captureFlow.run()
totalTime = int((time.time() - timeStart) * 1000)
ec = captureFlow.ExecutionContext
state = ec.get_state()
result = {
"id" : ec.Id,
"results" : ec.Results,
"logs" : ec.Logs,
"state" : state,
"fps" : captureFlow.Fps,
"release_frame" : captureFlow.ReleaseFrame,
"start_frame" : captureFlow.StartFrame,
"end_frame" : captureFlow.EndFrame,
"save_fps": captureFlow.SaveFps,
"trigger_offset" : captureFlow.TriggerInfo.TriggerTime,
"total_time" : totalTime
}
if ec.Error != None:
result["capture_error"] = ec.Error.AsObject()
captureFlow.save_logs()
return json.dumps(result)
except Exception as E:
request["error"] = str(E)
captureFlow.save_logs()
return json.dumps(request)
def get_arg(self, arg, isOptional = False):
req = flask.request.args
try:
val = req.get(arg)
return str(unquote(val))
except:
if isOptional:
return None
raise Exception("Could not find " + arg + " in the request")
def get_int(self, arg, isOptional = False):
try:
val = self.get_arg(arg)
return int(val)
except Exception as E:
if isOptional:
return None
raise E
def get_trigger_and_save_progress(self):
request = {}
try:
trigger_id = str(self.get_arg('id'))
exec_context = ExecutionContext(trigger_id, "/mnt/sdcard/DCIM/")
result = {}
result["id"] = trigger_id
state = None
timeBefore = time.time()
while state == None and time.time() - timeBefore < 1:
state = exec_context.get_state()
if state == None:
time.sleep(0.017)
result["state"] = state
return json.dumps(result)
except Exception as E:
request["error"] = str(E)
return json.dumps(request)
def ensure_camera_ready(self):
try:
camInfo = self.cam.get_storage_info()
if "available_space" not in camInfo:
raise CameraException(3, "StorageUnavailable")
if camInfo["available_space"] < 50000000:
raise CameraException(4, "StorageInsufficient")
self._validate_storage_writable()
except CameraException as camEx:
return json.dumps({ "Success": False, "Code": camEx.Code, "Message": camEx.Message })
except Exception as ex:
return json.dumps({ "Success": False, "Code": -1, "Message": str(ex) })
return json.dumps({ "Success": True, "Code": 0, "Message": None})
def _validate_storage_writable(self):
try:
timestamp = str(time.time())
fname = "/mnt/sdcard/DCIM/timestamp.txt"
storageFile = open(fname, "w")
storageFile.write(timestamp)
storageFile.close()
storageContent = open(fname, "r").readlines()[0]
if (storageContent != timestamp):
raise "could not write file"
except:
raise CameraException(2, "StorageReadonly")
The device is a radar unit and only allows for one camera to be synced at a given time. It reads the object it's tracking, tells the camera when to trigger and will save the file to ipad I am running the radar on. I would like to have a second camera trigger which would save the video on the second cameras interal memory. Doing this I can upload and merge with the other video file later. This would save me from manually having to trigger the second camera each time and trim the video lengths to match up. This would also allow me to connect the cameras wirelessly using travel routers. I was wondering how I could add the app_ext_multicast_trigger and multicast-trigger.conf to the app_ext_b1 python script so when the radar unit runs the app_ext_b1 python script on the camera it will also send out a multicast network packet to trigger camera two?
Related
This error usually should not appear, but recently when I run this, this error appears, and I have no clue how I will fix it.
Please if Anyone can help me to fix this error on this code below:
import requests
import json
from time import sleep
global OFFSET
OFFSET = 0
botToken = ""
global requestURL
global sendURL
requestURL = "http://api.telegram.org/bot" + botToken + "/getUpdates"
sendURL = "http://api.telegram.org/bot" + botToken + "/sendMessage"
def update (url):
global OFFSET
try:
update_raw = requests.get(url + "?offset=" + str(OFFSET))
update = update_raw.json()
result = extract_result(update)
if result != False:
OFFSET = result['update_id'] + 1
return result
else:
return False
except requests.exceptions.ConnectionError:
pass
def extract_result (dict):
result_array = dict['result']
if result_array == []:
return False
else:
result_dic = result_array[0]
return result_dic
def is_callback (dict):
if 'callback_query' in dict:
return True
def send_message (chatId, message):
requests.post(sendURL + "?chat_id=" + str(chatId) + "&text=" + message)
def send_message_button (chatId, message, buttonJSON):
requests.post(sendURL + "?chat_id=" + str(chatId) + "&reply_markup=" + buttonJSON + "&text=" + message)
#print (sendURL + "?chat_id=" + str(chatId) + "&reply_markup=" + buttonJSON + "&text=" + message)
while True:
newmessage = update (requestURL)
if newmessage != False:
if is_callback(newmessage) == True:
userchatid = newmessage['callback_query']['message']['chat']['id']
usertext = newmessage['callback_query']['message']['text']
username = newmessage['callback_query']['message']['chat']['first_name']
callback_data = newmessage['callback_query']['data']
send_message (userchatid, "Callback from " + callback_data + ", pressed by " + username)
else:
userchatid = newmessage['message']['chat']['id']
usertext = newmessage['message']['text']
username = newmessage['message']['chat']['first_name']
if usertext.lower() == "button":
buttonDict1 = {"text":"Knopf\n" + "hitest", "callback_data":"Knopf"}
buttonDict2 = {"text":"Knopf2", "callback_data":"Knopf2"}
buttonArr = {"inline_keyboard":[[buttonDict1, buttonDict2]]}
send_message_button (userchatid, "Hi " + username, json.dumps(buttonArr))
else:
send_message(userchatid, "You said: " + usertext)
sleep (1)
This is the error that appears to me after I run this bot
Line: 67
userchatid = newmessage['message']['chat']['id']
KeyError: 'message'
You catch the requests.exceptions.ConnectionError but don't handle it ( in the update function ), so now update does not return False as it returns nothing at all and can pass your check and cause havock.
Try to deal with the exception, or at least put a print in there to see if it's the one causing you issues, good luck!
This script works perfectly on Mac OS and Linux but when I try it on Windows it does not work.
When I try to try it on Windows, this error appears...
I imagine the problem from switch_user(dev).
I tried it with her (dev) but I couldn't solve it.
Whereas when Check GPT is verified, it stops and the error occurs.
PS C:\Users\motc-pc\Desktop\amonet-karnak-\amonet\modules> python main.py
[2020-06-03 22:03:49.653199] Waiting for bootrom
[2020-06-03 22:03:59.339064] Found port = COM22
[2020-06-03 22:03:59.348800] Handshake
* * * If you have a short attached, remove it now * * *
* * * Press Enter to continue * * *
[2020-06-03 22:04:02.368897] Init crypto engine
[2020-06-03 22:04:02.422396] Disable caches
[2020-06-03 22:04:02.424535] Disable bootrom range checks
[2020-06-03 22:04:02.459386] Load payload from ../brom-payload/build/payload.bin = 0x4888 bytes
[2020-06-03 22:04:02.469524] Send payload
[2020-06-03 22:04:03.440416] Let's rock
[2020-06-03 22:04:03.442368] Wait for the payload to come online...
[2020-06-03 22:04:04.163004] all good
[2020-06-03 22:04:04.165239] Check GPT
Traceback (most recent call last):
File "main.py", line 450, in <module>
main()
File "main.py", line 361, in main
switch_user(dev)
File "main.py", line 321, in switch_user
block = dev.emmc_read(0)
File "main.py", line 196, in emmc_read
raise RuntimeError("read fail")
RuntimeError: read fail
This script is full.
As I said earlier when I trying it on Linux or MacOS it works normally without problems
import struct
import os
import sys
import time
from handshake import handshake
from load_payload import load_payload, UserInputThread
from logger import log
import struct
import glob
import serial
from logger import log
BAUD = 115200
TIMEOUT = 5
CRYPTO_BASE = 0x10210000 # for karnak
def serial_ports ():
""" Lists available serial ports
:raises EnvironmentError:
On unsupported or unknown platforms
:returns:
A set containing the serial ports available on the system
"""
if sys.platform.startswith("win"):
ports = [ "COM{0:d}".format(i + 1) for i in range(256) ]
elif sys.platform.startswith("linux"):
ports = glob.glob("/dev/ttyACM*")
elif sys.platform.startswith("darwin"):
ports = glob.glob("/dev/cu.usbmodem*")
else:
raise EnvironmentError("Unsupported platform")
result = set()
for port in ports:
try:
s = serial.Serial(port, timeout=TIMEOUT)
s.close()
result.add(port)
except (OSError, serial.SerialException):
pass
return result
def p32_be(x):
return struct.pack(">I", x)
class Device:
def __init__(self, port=None):
self.dev = None
if port:
self.dev = serial.Serial(port, BAUD, timeout=TIMEOUT)
def find_device(self,preloader=False):
if self.dev:
raise RuntimeError("Device already found")
if preloader:
log("Waiting for preloader")
else:
log("Waiting for bootrom")
old = serial_ports()
while True:
new = serial_ports()
# port added
if new > old:
port = (new - old).pop()
break
# port removed
elif old > new:
old = new
time.sleep(0.25)
log("Found port = {}".format(port))
self.dev = serial.Serial(port, BAUD, timeout=TIMEOUT)
def check(self, test, gold):
if test != gold:
raise RuntimeError("ERROR: Serial protocol mismatch")
def check_int(self, test, gold):
test = struct.unpack('>I', test)[0]
self.check(test, gold)
def _writeb(self, out_str):
self.dev.write(out_str)
return self.dev.read()
def handshake(self):
# look for start byte
while True:
c = self._writeb(b'\xa0')
if c == b'\x5f':
break
self.dev.flushInput()
# complete sequence
self.check(self._writeb(b'\x0a'), b'\xf5')
self.check(self._writeb(b'\x50'), b'\xaf')
self.check(self._writeb(b'\x05'), b'\xfa')
def handshake2(self, cmd='FACTFACT'):
# look for start byte
c = 0
while c != b'Y':
c = self.dev.read()
log("Preloader ready, sending " + cmd)
command = str.encode(cmd)
self.dev.write(command)
self.dev.flushInput()
def read32(self, addr, size=1):
result = []
self.dev.write(b'\xd1')
self.check(self.dev.read(1), b'\xd1') # echo cmd
self.dev.write(struct.pack('>I', addr))
self.check_int(self.dev.read(4), addr) # echo addr
self.dev.write(struct.pack('>I', size))
self.check_int(self.dev.read(4), size) # echo size
self.check(self.dev.read(2), b'\x00\x00') # arg check
for _ in range(size):
data = struct.unpack('>I', self.dev.read(4))[0]
result.append(data)
self.check(self.dev.read(2), b'\x00\x00') # status
# support scalar
if len(result) == 1:
return result[0]
else:
return result
def write32(self, addr, words, status_check=True):
# support scalar
if not isinstance(words, list):
words = [ words ]
self.dev.write(b'\xd4')
self.check(self.dev.read(1), b'\xd4') # echo cmd
self.dev.write(struct.pack('>I', addr))
self.check_int(self.dev.read(4), addr) # echo addr
self.dev.write(struct.pack('>I', len(words)))
self.check_int(self.dev.read(4), len(words)) # echo size
self.check(self.dev.read(2), b'\x00\x01') # arg check
for word in words:
self.dev.write(struct.pack('>I', word))
self.check_int(self.dev.read(4), word) # echo word
if status_check:
self.check(self.dev.read(2), b'\x00\x01') # status
def run_ext_cmd(self, cmd):
self.dev.write(b'\xC8')
self.check(self.dev.read(1), b'\xC8') # echo cmd
cmd = bytes([cmd])
self.dev.write(cmd)
self.check(self.dev.read(1), cmd)
self.dev.read(1)
self.dev.read(2)
def wait_payload(self):
data = self.dev.read(4)
if data != b"\xB1\xB2\xB3\xB4":
raise RuntimeError("received {} instead of expected pattern".format(data))
def emmc_read(self, idx):
# magic
self.dev.write(p32_be(0xf00dd00d))
# cmd
self.dev.write(p32_be(0x1000))
# block to read
self.dev.write(p32_be(idx))
data = self.dev.read(0x200)
if len(data) != 0x200:
raise RuntimeError("read fail")
return data
def emmc_write(self, idx, data):
if len(data) != 0x200:
raise RuntimeError("data must be 0x200 bytes")
# magic
self.dev.write(p32_be(0xf00dd00d))
# cmd
self.dev.write(p32_be(0x1001))
# block to write
self.dev.write(p32_be(idx))
# data
self.dev.write(data)
code = self.dev.read(4)
if code != b"\xd0\xd0\xd0\xd0":
raise RuntimeError("device failure")
def emmc_switch(self, part):
# magic
self.dev.write(p32_be(0xf00dd00d))
# cmd
self.dev.write(p32_be(0x1002))
# partition
self.dev.write(p32_be(part))
def reboot(self):
# magic
self.dev.write(p32_be(0xf00dd00d))
# cmd
self.dev.write(p32_be(0x3000))
def kick_watchdog(self):
# magic
self.dev.write(p32_be(0xf00dd00d))
# cmd
self.dev.write(p32_be(0x3001))
def rpmb_read(self):
# magic
self.dev.write(p32_be(0xf00dd00d))
# cmd
self.dev.write(p32_be(0x2000))
data = self.dev.read(0x100)
if len(data) != 0x100:
raise RuntimeError("read fail")
return data
def rpmb_write(self, data):
if len(data) != 0x100:
raise RuntimeError("data must be 0x100 bytes")
# magic
self.dev.write(p32_be(0xf00dd00d))
# cmd
self.dev.write(p32_be(0x2001))
# data
self.dev.write(data)
def switch_boot0(dev):
dev.emmc_switch(1)
block = dev.emmc_read(0)
if block[0:9] != b"EMMC_BOOT" and block != b"\x00" * 0x200:
dev.reboot()
raise RuntimeError("what's wrong with your BOOT0?")
dev.kick_watchdog()
def flash_data(dev, data, start_block, max_size=0):
while len(data) % 0x200 != 0:
data += b"\x00"
if max_size and len(data) > max_size:
raise RuntimeError("data too big to flash")
blocks = len(data) // 0x200
for x in range(blocks):
print("[{} / {}]".format(x + 1, blocks), end='\r')
dev.emmc_write(start_block + x, data[x * 0x200:(x + 1) * 0x200])
if x % 10 == 0:
dev.kick_watchdog()
print("")
def flash_binary(dev, path, start_block, max_size=0):
with open(path, "rb") as fin:
data = fin.read()
while len(data) % 0x200 != 0:
data += b"\x00"
flash_data(dev, data, start_block, max_size=0)
def dump_binary(dev, path, start_block, max_size=0):
with open(path, "w+b") as fout:
blocks = max_size // 0x200
for x in range(blocks):
print("[{} / {}]".format(x + 1, blocks), end='\r')
fout.write(dev.emmc_read(start_block + x))
if x % 10 == 0:
dev.kick_watchdog()
print("")
def force_fastboot(dev, gpt):
switch_user(dev)
block = list(dev.emmc_read(gpt["MISC"][0]))
block[0:16] = "FASTBOOT_PLEASE\x00".encode("utf-8")
dev.emmc_write(gpt["MISC"][0], bytes(block))
block = dev.emmc_read(gpt["MISC"][0])
def force_recovery(dev, gpt):
switch_user(dev)
block = list(dev.emmc_read(gpt["MISC"][0]))
block[0:16] = "boot-recovery\x00\x00\x00".encode("utf-8")
dev.emmc_write(gpt["MISC"][0], bytes(block))
block = dev.emmc_read(gpt["MISC"][0])
def switch_user(dev):
dev.emmc_switch(0)
block = dev.emmc_read(0)
dev.kick_watchdog()
def parse_gpt(dev):
data = dev.emmc_read(0x400 // 0x200) + dev.emmc_read(0x600 // 0x200) + dev.emmc_read(0x800 // 0x200) + dev.emmc_read(0xA00 // 0x200)
num = len(data) // 0x80
parts = dict()
for x in range(num):
part = data[x * 0x80:(x + 1) * 0x80]
part_name = part[0x38:].decode("utf-16le").rstrip("\x00")
part_start = struct.unpack("<Q", part[0x20:0x28])[0]
part_end = struct.unpack("<Q", part[0x28:0x30])[0]
parts[part_name] = (part_start, part_end - part_start + 1)
return parts
def main():
minimal = False
dev = Device()
dev.find_device()
# 0.1) Handshake
handshake(dev)
# 0.2) Load brom payload
load_payload(dev, "../brom-payload/build/payload.bin")
dev.kick_watchdog()
if len(sys.argv) == 2 and sys.argv[1] == "minimal":
thread = UserInputThread(msg = "Running in minimal mode, assuming LK, TZ, LK-payload and TWRP to have already been flashed.\nIf this is correct (i.e. you used \"brick\" option in step 1) press enter, otherwise terminate with Ctrl+C")
thread.start()
while not thread.done:
dev.kick_watchdog()
time.sleep(1)
minimal = True
# 1) Sanity check GPT
log("Check GPT")
switch_user(dev)
# 1.1) Parse gpt
gpt = parse_gpt(dev)
log("gpt_parsed = {}".format(gpt))
if "lk" not in gpt or "tee1" not in gpt or "boot" not in gpt or "recovery" not in gpt:
raise RuntimeError("bad gpt")
# 2) Sanity check boot0
log("Check boot0")
switch_boot0(dev)
# 3) Sanity check rpmb
log("Check rpmb")
rpmb = dev.rpmb_read()
if rpmb[0:4] != b"AMZN":
thread = UserInputThread(msg = "rpmb looks broken; if this is expected (i.e. you're retrying the exploit) press enter, otherwise terminate with Ctrl+C")
thread.start()
while not thread.done:
dev.kick_watchdog()
time.sleep(1)
# Clear preloader so, we get into bootrom without shorting, should the script stall (we flash preloader as last step)
# 4) Downgrade preloader
log("Clear preloader header")
switch_boot0(dev)
flash_data(dev, b"EMMC_BOOT" + b"\x00" * ((0x200 * 8) - 9), 0)
# 5) Zero out rpmb to enable downgrade
log("Downgrade rpmb")
dev.rpmb_write(b"\x00" * 0x100)
log("Recheck rpmb")
rpmb = dev.rpmb_read()
if rpmb != b"\x00" * 0x100:
dev.reboot()
raise RuntimeError("downgrade failure, giving up")
log("rpmb downgrade ok")
dev.kick_watchdog()
if not minimal:
# 6) Install preloader
log("Flash preloader")
switch_boot0(dev)
flash_binary(dev, "../bin/preloader.bin", 8)
flash_binary(dev, "../bin/preloader.bin", 520)
# 6) Install lk-payload
log("Flash lk-payload")
switch_boot0(dev)
flash_binary(dev, "../lk-payload/build/payload.bin", 1024)
# 7) Downgrade tz
log("Flash tz")
switch_user(dev)
flash_binary(dev, "../bin/tz.img", gpt["tee1"][0], gpt["tee1"][1] * 0x200)
# 8) Downgrade lk
log("Flash lk")
switch_user(dev)
flash_binary(dev, "../bin/lk.bin", gpt["lk"][0], gpt["lk"][1] * 0x200)
# 9) Flash microloader
log("Inject microloader")
switch_user(dev)
boot_hdr1 = dev.emmc_read(gpt["boot"][0]) + dev.emmc_read(gpt["boot"][0] + 1)
boot_hdr2 = dev.emmc_read(gpt["boot"][0] + 2) + dev.emmc_read(gpt["boot"][0] + 3)
flash_binary(dev, "../bin/microloader.bin", gpt["boot"][0], 2 * 0x200)
if boot_hdr2[0:8] != b"ANDROID!":
flash_data(dev, boot_hdr1, gpt["boot"][0] + 2, 2 * 0x200)
if not minimal:
log("Force fastboot")
force_fastboot(dev, gpt)
else:
log("Force recovery")
force_recovery(dev, gpt)
# 10) Downgrade preloader
log("Flash preloader header")
switch_boot0(dev)
flash_binary(dev, "../bin/preloader.hdr0", 0, 4)
flash_binary(dev, "../bin/preloader.hdr1", 4, 4)
# Reboot (to fastboot or recovery)
log("Reboot")
dev.reboot()
if __name__ == "__main__":
main()
I am trying to use unity to simulate a real world location and movement of a drone while simutaniously moving the DJI tello Drone in the real world environment.
I am getting the error:
Traceback (most recent call last):
File "<ipython-input-6-6b86f3bafb33>", line 1, in <module>
runfile('C:/Users/fahre/Desktop/dronecontrol.py', wdir='C:/Users/fahre/Desktop')
File "C:\ProgramData\Anaconda2\lib\site-packages\spyder_kernels\customize\spydercustomize.py", line 827, in runfile
execfile(filename, namespace)
File "C:\ProgramData\Anaconda2\lib\site-packages\spyder_kernels\customize\spydercustomize.py", line 95, in execfile
exec(compile(scripttext, filename, 'exec'), glob, loc)
File "C:/Users/fahre/Desktop/dronecontrol.py", line 73, in <module>
tellodrone = tellopy.Tello()
File "C:\ProgramData\Anaconda2\lib\site-packages\tellopy\_internal\tello.py", line 91, in __init__
self.sock.bind(('', self.port))
File "C:\ProgramData\Anaconda2\lib\socket.py", line 228, in meth
return getattr(self._sock,name)(*args)
error: [Errno 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted
some suggested that I use
#s.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
in the tello.py source file but im not sure where to do the edit.
# Create a UDP socket
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.bind(('', self.port))
self.sock.settimeout(2.0)
most of my tello.py file code will be posted below
import threading
import socket
import time
import datetime
import struct
import sys
import os
from import crc
from import logger
from import event
from import state
from import error
from import video_stream
from utils import *
from protocol import *
from import dispatcher
log = logger.Logger('Tello')
class Tello(object):
EVENT_CONNECTED = event.Event('connected')
EVENT_WIFI = event.Event('wifi')
EVENT_LIGHT = event.Event('light')
EVENT_FLIGHT_DATA = event.Event('fligt_data')
EVENT_LOG = event.Event('log')
EVENT_TIME = event.Event('time')
EVENT_VIDEO_FRAME = event.Event('video frame')
EVENT_VIDEO_DATA = event.Event('video data')
EVENT_DISCONNECTED = event.Event('disconnected')
EVENT_FILE_RECEIVED = event.Event('file received')
# internal events
__EVENT_CONN_REQ = event.Event('conn_req')
__EVENT_CONN_ACK = event.Event('conn_ack')
__EVENT_TIMEOUT = event.Event('timeout')
__EVENT_QUIT_REQ = event.Event('quit_req')
# for backward comaptibility
CONNECTED_EVENT = EVENT_CONNECTED
WIFI_EVENT = EVENT_WIFI
LIGHT_EVENT = EVENT_LIGHT
FLIGHT_EVENT = EVENT_FLIGHT_DATA
LOG_EVENT = EVENT_LOG
TIME_EVENT = EVENT_TIME
VIDEO_FRAME_EVENT = EVENT_VIDEO_FRAME
STATE_DISCONNECTED = state.State('disconnected')
STATE_CONNECTING = state.State('connecting')
STATE_CONNECTED = state.State('connected')
STATE_QUIT = state.State('quit')
LOG_ERROR = logger.LOG_ERROR
LOG_WARN = logger.LOG_WARN
LOG_INFO = logger.LOG_INFO
LOG_DEBUG = logger.LOG_DEBUG
LOG_ALL = logger.LOG_ALL
def __init__(self, port=9000):
self.tello_addr = ('192.168.10.1', 8889)
self.debug = False
self.pkt_seq_num = 0x01e4
self.port = port
self.udpsize = 2000
self.left_x = 0.0
self.left_y = 0.0
self.right_x = 0.0
self.right_y = 0.0
self.sock = None
self.state = self.STATE_DISCONNECTED
self.lock = threading.Lock()
self.connected = threading.Event()
self.video_enabled = False
self.prev_video_data_time = None
self.video_data_size = 0
self.video_data_loss = 0
self.log = log
self.exposure = 0
self.video_encoder_rate = 4
self.video_stream = None
self.wifi_strength = 0
# video zoom state
self.zoom = False
# File recieve state.
self.file_recv = {} # Map filenum -> protocol.DownloadedFile
#s.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
# Create a UDP socket
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.bind(('', self.port))
self.sock.settimeout(2.0)
dispatcher.connect(self.__state_machine, dispatcher.signal.All)
threading.Thread(target=self.__recv_thread).start()
threading.Thread(target=self.__video_thread).start()
def set_loglevel(self, level):
"""
Set_loglevel controls the output messages. Valid levels are
LOG_ERROR, LOG_WARN, LOG_INFO, LOG_DEBUG and LOG_ALL.
"""
log.set_level(level)
def get_video_stream(self):
"""
Get_video_stream is used to prepare buffer object which receive video data from the drone.
"""
newly_created = False
self.lock.acquire()
log.info('get video stream')
try:
if self.video_stream is None:
self.video_stream = video_stream.VideoStream(self)
newly_created = True
res = self.video_stream
finally:
self.lock.release()
if newly_created:
self.__send_exposure()
self.__send_video_encoder_rate()
self.start_video()
return res
def connect(self):
"""Connect is used to send the initial connection request to the drone."""
self.__publish(event=self.__EVENT_CONN_REQ)
def wait_for_connection(self, timeout=None):
"""Wait_for_connection will block until the connection is established."""
if not self.connected.wait(timeout):
raise error.TelloError('timeout')
def __send_conn_req(self):
port = 9617
port0 = (int(port/1000) % 10) << 4 | (int(port/100) % 10)
port1 = (int(port/10) % 10) << 4 | (int(port/1) % 10)
buf = 'conn_req:%c%c' % (chr(port0), chr(port1))
log.info('send connection request (cmd="%s%02x%02x")' % (str(buf[:-2]), port0, port1))
return self.send_packet(Packet(buf))
def subscribe(self, signal, handler):
"""Subscribe a event such as EVENT_CONNECTED, EVENT_FLIGHT_DATA, EVENT_VIDEO_FRAME and so on."""
dispatcher.connect(handler, signal)
def __publish(self, event, data=None, **args):
args.update({'data': data})
if 'signal' in args:
del args['signal']
if 'sender' in args:
del args['sender']
log.debug('publish signal=%s, args=%s' % (event, args))
dispatcher.send(event, sender=self, **args)
def takeoff(self):
"""Takeoff tells the drones to liftoff and start flying."""
log.info('set altitude limit 30m')
pkt = Packet(SET_ALT_LIMIT_CMD)
pkt.add_byte(0x1e) # 30m
pkt.add_byte(0x00)
self.send_packet(pkt)
log.info('takeoff (cmd=0x%02x seq=0x%04x)' % (TAKEOFF_CMD, self.pkt_seq_num))
pkt = Packet(TAKEOFF_CMD)
pkt.fixup()
return self.send_packet(pkt)
def land(self):
"""Land tells the drone to come in for landing."""
log.info('land (cmd=0x%02x seq=0x%04x)' % (LAND_CMD, self.pkt_seq_num))
pkt = Packet(LAND_CMD)
pkt.add_byte(0x00)
pkt.fixup()
return self.send_packet(pkt)
def palm_land(self):
"""Tells the drone to wait for a hand underneath it and then land."""
log.info('palmland (cmd=0x%02x seq=0x%04x)' % (PALM_LAND_CMD, self.pkt_seq_num))
pkt = Packet(PALM_LAND_CMD)
pkt.add_byte(0x00)
pkt.fixup()
return self.send_packet(pkt)
def quit(self):
"""Quit stops the internal threads."""
log.info('quit')
self.__publish(event=self.__EVENT_QUIT_REQ)
def __send_time_command(self):
log.info('send_time (cmd=0x%02x seq=0x%04x)' % (TIME_CMD, self.pkt_seq_num))
pkt = Packet(TIME_CMD, 0x50)
pkt.add_byte(0)
pkt.add_time()
pkt.fixup()
return self.send_packet(pkt)
def __send_start_video(self):
pkt = Packet(VIDEO_START_CMD, 0x60)
pkt.fixup()
return self.send_packet(pkt)
def __send_video_mode(self, mode):
pkt = Packet(VIDEO_MODE_CMD)
pkt.add_byte(mode)
pkt.fixup()
return self.send_packet(pkt)
def set_video_mode(self, zoom=False):
"""Tell the drone whether to capture 960x720 4:3 video, or 1280x720 16:9 zoomed video.
4:3 has a wider field of view (both vertically and horizontally), 16:9 is crisper."""
log.info('set video mode zoom=%s (cmd=0x%02x seq=0x%04x)' % (
zoom, VIDEO_START_CMD, self.pkt_seq_num))
self.zoom = zoom
return self.__send_video_mode(int(zoom))
def start_video(self):
"""Start_video tells the drone to send start info (SPS/PPS) for video stream."""
log.info('start video (cmd=0x%02x seq=0x%04x)' % (VIDEO_START_CMD, self.pkt_seq_num))
self.video_enabled = True
self.__send_exposure()
self.__send_video_encoder_rate()
return self.__send_start_video()
def set_exposure(self, level):
"""Set_exposure sets the drone camera exposure level. Valid levels are 0, 1, and 2."""
if level < 0 or 2 < level:
raise error.TelloError('Invalid exposure level')
log.info('set exposure (cmd=0x%02x seq=0x%04x)' % (EXPOSURE_CMD, self.pkt_seq_num))
self.exposure = level
return self.__send_exposure()
def __send_exposure(self):
pkt = Packet(EXPOSURE_CMD, 0x48)
pkt.add_byte(self.exposure)
pkt.fixup()
return self.send_packet(pkt)
def set_video_encoder_rate(self, rate):
"""Set_video_encoder_rate sets the drone video encoder rate."""
log.info('set video encoder rate (cmd=0x%02x seq=%04x)' %
(VIDEO_ENCODER_RATE_CMD, self.pkt_seq_num))
self.video_encoder_rate = rate
return self.__send_video_encoder_rate()
def __send_video_encoder_rate(self):
pkt = Packet(VIDEO_ENCODER_RATE_CMD, 0x68)
pkt.add_byte(self.video_encoder_rate)
pkt.fixup()
return self.send_packet(pkt)
def take_picture(self):
log.info('take picture')
return self.send_packet_data(TAKE_PICTURE_COMMAND, type=0x68)
def up(self, val):
"""Up tells the drone to ascend. Pass in an int from 0-100."""
log.info('up(val=%d)' % val)
self.left_y = val / 100.0
def down(self, val):
"""Down tells the drone to descend. Pass in an int from 0-100."""
log.info('down(val=%d)' % val)
self.left_y = val / 100.0 * -1
def forward(self, val):
"""Forward tells the drone to go forward. Pass in an int from 0-100."""
log.info('forward(val=%d)' % val)
self.right_y = val / 100.0
def backward(self, val):
"""Backward tells the drone to go in reverse. Pass in an int from 0-100."""
log.info('backward(val=%d)' % val)
self.right_y = val / 100.0 * -1
def right(self, val):
"""Right tells the drone to go right. Pass in an int from 0-100."""
log.info('right(val=%d)' % val)
self.right_x = val / 100.0
def left(self, val):
"""Left tells the drone to go left. Pass in an int from 0-100."""
log.info('left(val=%d)' % val)
self.right_x = val / 100.0 * -1
def clockwise(self, val):
"""
Clockwise tells the drone to rotate in a clockwise direction.
Pass in an int from 0-100.
"""
log.info('clockwise(val=%d)' % val)
self.left_x = val / 100.0
def counter_clockwise(self, val):
"""
CounterClockwise tells the drone to rotate in a counter-clockwise direction.
Pass in an int from 0-100.
"""
log.info('counter_clockwise(val=%d)' % val)
self.left_x = val / 100.0 * -1
def flip_forward(self):
"""flip_forward tells the drone to perform a forwards flip"""
log.info('flip_forward (cmd=0x%02x seq=0x%04x)' % (FLIP_CMD, self.pkt_seq_num))
pkt = Packet(FLIP_CMD, 0x70)
pkt.add_byte(FlipFront)
pkt.fixup()
return self.send_packet(pkt)
def flip_back(self):
"""flip_back tells the drone to perform a backwards flip"""
log.info('flip_back (cmd=0x%02x seq=0x%04x)' % (FLIP_CMD, self.pkt_seq_num))
pkt = Packet(FLIP_CMD, 0x70)
pkt.add_byte(FlipBack)
pkt.fixup()
return self.send_packet(pkt)
def flip_right(self):
"""flip_right tells the drone to perform a right flip"""
log.info('flip_right (cmd=0x%02x seq=0x%04x)' % (FLIP_CMD, self.pkt_seq_num))
pkt = Packet(FLIP_CMD, 0x70)
pkt.add_byte(FlipRight)
pkt.fixup()
return self.send_packet(pkt)
def flip_left(self):
"""flip_left tells the drone to perform a left flip"""
log.info('flip_left (cmd=0x%02x seq=0x%04x)' % (FLIP_CMD, self.pkt_seq_num))
pkt = Packet(FLIP_CMD, 0x70)
pkt.add_byte(FlipLeft)
pkt.fixup()
return self.send_packet(pkt)
def flip_forwardleft(self):
"""flip_forwardleft tells the drone to perform a forwards left flip"""
log.info('flip_forwardleft (cmd=0x%02x seq=0x%04x)' % (FLIP_CMD, self.pkt_seq_num))
pkt = Packet(FLIP_CMD, 0x70)
pkt.add_byte(FlipForwardLeft)
pkt.fixup()
return self.send_packet(pkt)
def flip_backleft(self):
"""flip_backleft tells the drone to perform a backwards left flip"""
log.info('flip_backleft (cmd=0x%02x seq=0x%04x)' % (FLIP_CMD, self.pkt_seq_num))
pkt = Packet(FLIP_CMD, 0x70)
pkt.add_byte(FlipBackLeft)
pkt.fixup()
return self.send_packet(pkt)
def flip_forwardright(self):
"""flip_forwardright tells the drone to perform a forwards right flip"""
log.info('flip_forwardright (cmd=0x%02x seq=0x%04x)' % (FLIP_CMD, self.pkt_seq_num))
pkt = Packet(FLIP_CMD, 0x70)
pkt.add_byte(FlipForwardRight)
pkt.fixup()
return self.send_packet(pkt)
def flip_backright(self):
"""flip_backleft tells the drone to perform a backwards right flip"""
log.info('flip_backright (cmd=0x%02x seq=0x%04x)' % (FLIP_CMD, self.pkt_seq_num))
pkt = Packet(FLIP_CMD, 0x70)
pkt.add_byte(FlipBackLeft)
pkt.fixup()
return self.send_packet(pkt)
def __fix_range(self, val, min=-1.0, max=1.0):
if val < min:
val = min
elif val > max:
val = max
return val
def set_throttle(self, throttle):
"""
Set_throttle controls the vertical up and down motion of the drone.
Pass in an int from -1.0 ~ 1.0. (positive value means upward)
"""
if self.left_y != self.__fix_range(throttle):
log.info('set_throttle(val=%4.2f)' % throttle)
self.left_y = self.__fix_range(throttle)
def set_yaw(self, yaw):
"""
Set_yaw controls the left and right rotation of the drone.
Pass in an int from -1.0 ~ 1.0. (positive value will make the drone turn to the right)
"""
if self.left_x != self.__fix_range(yaw):
log.info('set_yaw(val=%4.2f)' % yaw)
self.left_x = self.__fix_range(yaw)
def set_pitch(self, pitch):
"""
Set_pitch controls the forward and backward tilt of the drone.
Pass in an int from -1.0 ~ 1.0. (positive value will make the drone move forward)
"""
if self.right_y != self.__fix_range(pitch):
log.info('set_pitch(val=%4.2f)' % pitch)
self.right_y = self.__fix_range(pitch)
def set_roll(self, roll):
"""
Set_roll controls the the side to side tilt of the drone.
Pass in an int from -1.0 ~ 1.0. (positive value will make the drone move to the right)
"""
if self.right_x != self.__fix_range(roll):
log.info('set_roll(val=%4.2f)' % roll)
self.right_x = self.__fix_range(roll)
def __send_stick_command(self):
pkt = Packet(STICK_CMD, 0x60)
axis1 = int(1024 + 660.0 * self.right_x) & 0x7ff
axis2 = int(1024 + 660.0 * self.right_y) & 0x7ff
axis3 = int(1024 + 660.0 * self.left_y) & 0x7ff
axis4 = int(1024 + 660.0 * self.left_x) & 0x7ff
'''
11 bits (-1024 ~ +1023) x 4 axis = 44 bits
44 bits will be packed in to 6 bytes (48 bits)
axis4 axis3 axis2 axis1
| | | | |
4 3 2 1 0
98765432109876543210987654321098765432109876543210
| | | | | | |
byte5 byte4 byte3 byte2 byte1 byte0
'''
log.debug("stick command: yaw=%4d thr=%4d pit=%4d rol=%4d" %
(axis4, axis3, axis2, axis1))
log.debug("stick command: yaw=%04x thr=%04x pit=%04x rol=%04x" %
(axis4, axis3, axis2, axis1))
pkt.add_byte(((axis2 << 11 | axis1) >> 0) & 0xff)
pkt.add_byte(((axis2 << 11 | axis1) >> 8) & 0xff)
pkt.add_byte(((axis3 << 11 | axis2) >> 5) & 0xff)
pkt.add_byte(((axis4 << 11 | axis3) >> 2) & 0xff)
pkt.add_byte(((axis4 << 11 | axis3) >> 10) & 0xff)
pkt.add_byte(((axis4 << 11 | axis3) >> 18) & 0xff)
pkt.add_time()
pkt.fixup()
log.debug("stick command: %s" % byte_to_hexstring(pkt.get_buffer()))
return self.send_packet(pkt)
def send_packet(self, pkt):
"""Send_packet is used to send a command packet to the drone."""
try:
cmd = pkt.get_buffer()
self.sock.sendto(cmd, self.tello_addr)
log.debug("send_packet: %s" % byte_to_hexstring(cmd))
except socket.error as err:
if self.state == self.STATE_CONNECTED:
log.error("send_packet: %s" % str(err))
else:
log.info("send_packet: %s" % str(err))
return False
return True
def send_packet_data(self, command, type=0x68, payload=[]):
pkt = Packet(command, type, payload)
pkt.fixup()
return self.send_packet(pkt)
def __process_packet(self, data):
if isinstance(data, str):
data = bytearray([x for x in data])
if str(data[0:9]) == 'conn_ack:' or data[0:9] == b'conn_ack:':
log.info('connected. (port=%2x%2x)' % (data[9], data[10]))
log.debug(' %s' % byte_to_hexstring(data))
if self.video_enabled:
self.__send_exposure()
self.__send_video_encoder_rate()
self.__send_start_video()
self.__publish(self.__EVENT_CONN_ACK, data)
return True
if data[0] != START_OF_PACKET:
log.info('start of packet != %02x (%02x) (ignored)' % (START_OF_PACKET, data[0]))
log.info(' %s' % byte_to_hexstring(data))
log.info(' %s' % str(map(chr, data))[1:-1])
return False
pkt = Packet(data)
cmd = uint16(data[5], data[6])
if cmd == LOG_MSG:
log.debug("recv: log: %s" % byte_to_hexstring(data[9:]))
self.__publish(event=self.EVENT_LOG, data=data[9:])
elif cmd == WIFI_MSG:
log.debug("recv: wifi: %s" % byte_to_hexstring(data[9:]))
self.wifi_strength = data[9]
self.__publish(event=self.EVENT_WIFI, data=data[9:])
elif cmd == LIGHT_MSG:
log.debug("recv: light: %s" % byte_to_hexstring(data[9:]))
self.__publish(event=self.EVENT_LIGHT, data=data[9:])
elif cmd == FLIGHT_MSG:
flight_data = FlightData(data[9:])
flight_data.wifi_strength = self.wifi_strength
log.debug("recv: flight data: %s" % str(flight_data))
self.__publish(event=self.EVENT_FLIGHT_DATA, data=flight_data)
elif cmd == TIME_CMD:
log.debug("recv: time data: %s" % byte_to_hexstring(data))
self.__publish(event=self.EVENT_TIME, data=data[7:9])
elif cmd in (TAKEOFF_CMD, LAND_CMD, VIDEO_START_CMD, VIDEO_ENCODER_RATE_CMD, PALM_LAND_CMD,
EXPOSURE_CMD):
log.info("recv: ack: cmd=0x%02x seq=0x%04x %s" %
(uint16(data[5], data[6]), uint16(data[7], data[8]), byte_to_hexstring(data)))
elif cmd == TELLO_CMD_FILE_SIZE:
# Drone is about to send us a file. Get ready.
# N.b. one of the fields in the packet is a file ID; by demuxing
# based on file ID we can receive multiple files at once. This
# code doesn't support that yet, though, so don't take one photo
# while another is still being received.
log.info("recv: file size: %s" % byte_to_hexstring(data))
if len(pkt.get_data()) >= 7:
(size, filenum) = struct.unpack('<xLH', pkt.get_data())
log.info(' file size: num=%d bytes=%d' % (filenum, size))
# Initialize file download state.
self.file_recv[filenum] = DownloadedFile(filenum, size)
else:
# We always seem to get two files, one with most of the payload missing.
# Not sure what the second one is for.
log.warn(' file size: payload too small: %s' % byte_to_hexstring(pkt.get_data()))
# Ack the packet.
self.send_packet(pkt)
elif cmd == TELLO_CMD_FILE_DATA:
# log.info("recv: file data: %s" % byte_to_hexstring(data[9:21]))
# Drone is sending us a fragment of a file it told us to prepare
# for earlier.
self.recv_file_data(pkt.get_data())
else:
log.info('unknown packet: %04x %s' % (cmd, byte_to_hexstring(data)))
return False
return True
def recv_file_data(self, data):
(filenum,chunk,fragment,size) = struct.unpack('<HLLH', data[0:12])
file = self.file_recv.get(filenum, None)
# Preconditions.
if file is None:
return
if file.recvFragment(chunk, fragment, size, data[12:12+size]):
# Did this complete a chunk? Ack the chunk so the drone won't
# re-send it.
self.send_packet_data(TELLO_CMD_FILE_DATA, type=0x50,
payload=struct.pack('<BHL', 0, filenum, chunk))
if file.done():
# We have the whole file! First, send a normal ack with the first
# byte set to 1 to indicate file completion.
self.send_packet_data(TELLO_CMD_FILE_DATA, type=0x50,
payload=struct.pack('<BHL', 1, filenum, chunk))
# Then send the FILE_COMPLETE packed separately telling it how
# large we thought the file was.
self.send_packet_data(TELLO_CMD_FILE_COMPLETE, type=0x48,
payload=struct.pack('<HL', filenum, file.size))
# Inform subscribers that we have a file and clean up.
self.__publish(event=self.EVENT_FILE_RECEIVED, data=file.data())
del self.file_recv[filenum]
def __state_machine(self, event, sender, data, **args):
self.lock.acquire()
cur_state = self.state
event_connected = False
event_disconnected = False
log.debug('event %s in state %s' % (str(event), str(self.state)))
if self.state == self.STATE_DISCONNECTED:
if event == self.__EVENT_CONN_REQ:
self.__send_conn_req()
self.state = self.STATE_CONNECTING
elif event == self.__EVENT_QUIT_REQ:
self.state = self.STATE_QUIT
event_disconnected = True
self.video_enabled = False
elif self.state == self.STATE_CONNECTING:
if event == self.__EVENT_CONN_ACK:
self.state = self.STATE_CONNECTED
event_connected = True
# send time
self.__send_time_command()
elif event == self.__EVENT_TIMEOUT:
self.__send_conn_req()
elif event == self.__EVENT_QUIT_REQ:
self.state = self.STATE_QUIT
elif self.state == self.STATE_CONNECTED:
if event == self.__EVENT_TIMEOUT:
self.__send_conn_req()
self.state = self.STATE_CONNECTING
event_disconnected = True
self.video_enabled = False
elif event == self.__EVENT_QUIT_REQ:
self.state = self.STATE_QUIT
event_disconnected = True
self.video_enabled = False
elif self.state == self.STATE_QUIT:
pass
if cur_state != self.state:
log.info('state transit %s -> %s' % (cur_state, self.state))
self.lock.release()
if event_connected:
self.__publish(event=self.EVENT_CONNECTED, **args)
self.connected.set()
if event_disconnected:
self.__publish(event=self.EVENT_DISCONNECTED, **args)
self.connected.clear()
def __recv_thread(self):
sock = self.sock
while self.state != self.STATE_QUIT:
if self.state == self.STATE_CONNECTED:
self.__send_stick_command() # ignore errors
try:
data, server = sock.recvfrom(self.udpsize)
log.debug("recv: %s" % byte_to_hexstring(data))
self.__process_packet(data)
except socket.timeout as ex:
if self.state == self.STATE_CONNECTED:
log.error('recv: timeout')
self.__publish(event=self.__EVENT_TIMEOUT)
except Exception as ex:
log.error('recv: %s' % str(ex))
show_exception(ex)
log.info('exit from the recv thread.')
def __video_thread(self):
log.info('start video thread')
# Create a UDP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
port = 6038
sock.bind(('', port))
sock.settimeout(5.0)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 512 * 1024)
log.info('video receive buffer size = %d' %
sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF))
prev_video_data = None
prev_ts = None
history = []
while self.state != self.STATE_QUIT:
if not self.video_enabled:
time.sleep(1.0)
continue
try:
data, server = sock.recvfrom(self.udpsize)
now = datetime.datetime.now()
log.debug("video recv: %s %d bytes" % (byte_to_hexstring(data[0:2]), len(data)))
show_history = False
I have a problem.
I have a page where I send commands to a sensor network.
When I click on this part of code
<a href='javascript:void(send_command_to_sensor("{{net.id}}", "{{sens.id}}", "identify"));'></a>
I call a js function, this:
function send_command_to_sensor(net, sens, command) {
$.ajax({url: "/networks/" + net + "/sensors/" + sens + "/send?command=" + command,
type: "GET",
async: true,
dataType: "json",
success: function(json_response) {
var err = json_response['error'];
if (err) {
show_alert('error', err);
return;
}
var success = json_response['success'];
if (success) {
show_alert('success', success);
return;
}
show_alert('alert', "This should not happen!");
}
});
}
This function build a url that recall an handler in the Tornado web server written in python. The handler is this:
################################################################################
# SensorSend handler.
# Sends a packet to the sensor by writing in a file (Should work through
# MySQL database) read by Quantaserv Daemon
################################################################################
class SensorSendHandler(BaseHandler):
# Requires authentication
#tornado.web.authenticated
def get(self, nid, sid):
command = self.get_argument('command').upper();
print command
# Retrieve the current user
usr = self.get_current_user()
usr_id = usr['id']
self.lock_tables("read", ['nets_permissions as n'])
perm = self.db.get("SELECT n.perm FROM nets_permissions as n \
WHERE n.network_id=%s AND n.user_id=%s", nid, int(usr_id))
self.unlock_tables()
# Check whether the user has access to the network
perms = self.check_network_access(nid, perm['perm'])
#Check wether the sensor exists in this network
self.check_sensor_in_network(sid, nid)
# The dictionary to return
ret = {}
############################################################################
# The following Code is valid for ZTC networks only
# Must be generalized
# TODO: - get the network type:
# - check wether the command is allowed
############################################################################
if command not in ['ON', 'OFF', 'TOGGLE', 'IDENTIFY']:
raise tornado.web.HTTPError(404, "Unknown command: " + str(command))
#Command OnOffCmd_SetState
if command in ['ON', 'OFF', 'TOGGLE']:
op_group = "70"
op_code = "50"
packet_meta = "%s%s%s%s02%s%s600080000%s"
pkt_len = hextransform(16, 2)
netid = hextransform(int(nid), 16)
sens_id = hextransform(int(sid) >> 16, 4)
sens_id_little = invert2bytes(sens_id,0)
cluster_id = hextransform(int(sid) & 0x00FFFF, 4)
end_point = "08"
if command == 'ON':
cmd_data = "01"
elif command == 'OFF':
cmd_data = "00"
elif command == 'TOGGLE':
cmd_data = "02"
packet = packet_meta % (netid, op_group, op_code, pkt_len, sens_id, end_point, cmd_data)
packet = packet.upper()
op_group_hex=0x70
op_code_hex=0x50
print command
#Command ZDP-IEEE_addr.Request
elif command == 'IDENTIFY':
op_group = "A2"
op_code = "01"
packet_meta = "%s%s%s%s"
pkt_len = hextransform(2, 2)
sens_id = hextransform(int(sid) >> 16, 4)
sens_id_little = invert2bytes(sens_id,0)
packet = packet_meta % (op_group, op_code, pkt_len, sens_id)
packet = packet.upper()
op_group_hex=0xA2
op_code_hex=0x01
#Command ZDP-Active_EP_req.Request
elif command == 'HOWMANY':
op_group = "A2"
op_code = "05"
packet_meta = "%s%s%s%s%s"
pkt_len = hextransform(4, 2)
netid = hextransform(int(nid), 16)
sens_id = hextransform(int(sid) >> 16, 4)
sens_id_little = invert2bytes(sens_id,0)
packet = packet_meta % (op_group, op_code, pkt_len, sens_id, netid)
packet = packet.upper()
op_group_hex=0xA2
op_code_hex=0x05
mynet_type ="ztc"
cmdjson = packet2json(op_group_hex,op_code_hex, packet)
print("\t\t " + packet + "\n")
#
#
#TODO : -write command into db
ts = datetime.datetime.now().isoformat()
self.lock_tables("write", ['confcommands'])
self.db.execute("INSERT INTO confcommands (network_id, ntype, timestamp, command) \
VALUES (%s,%s,%s,%s)", nid, mynet_type, ts, cmdjson)
self.unlock_tables();
############### ELISA ##########################################
# TODO: - open the /tmp/iztc file in append mode
cmdfile = open('/tmp/iztc', 'a')
# - acquire a lock "only for the DB case, it's easier"
# - write the packet
cmdfile.write(nid + "\t"+ mynet_type + "\t"+ ts + "\t"+ cmdjson +"\n");
# - release the lock "only for the DB case, it's easier"
# - close the file
cmdfile.close()
if command == 'HOWMANY':
opcodegroupr = "A0"
opcoder = "85"
elif command == 'IDENTIFY':
opcodegroupr = "A0"
opcoder = "81"
print command
#Code for retrieving the MAC address of the node
como_url = "".join(['http://', options.como_address, ':', options.como_port,
'/', ztc_config, '?netid=', netid,
'&opcode_group=', opcodegroupr,
'&opcode=', opcoder, '&start=-5m&end=-1s'])
http_client = AsyncHTTPClient()
response = yield tornado.gen.Task(http_client.fetch, como_url)
ret = {}
if response.error:
ret['error'] = 'Error while retrieving unregistered sensors'
else:
for line in response.body.split("\n"):
if line != "":
value = int(line.split(" ")[6])
ret['response'] = value
self.write(tornado.escape.json_encode(ret))
if command == 'HOWMANY':
status = value[0]
NwkAddress = value[1:2]
ActiveEPCount = value[3]
ActiveEPList = value[len(ActiveEPCount)]
if status == "0":
ret['success'] = "The %s command has been succesfully sent!" % (command.upper())
self.write(tornado.escape.json_encode(ret))
elif status == "80":
ret['error'] = "Invalid Request Type"
self.write(tornado.escape.json_encode(ret))
elif status == "89":
ret['error'] = "No Descriptor"
self.write(tornado.escape.json_encode(ret))
else:
ret['error'] = "Device not found!"
self.write(tornado.escape.json_encode(ret))
if command == 'IDENTIFY':
status = value[0]
IEEEAddrRemoteDev = value[1:8]
NWKAddrRemoteDev = value[9:2]
NumOfAssociatedDevice = value[11:1]
StartIndex = value[12:1]
ListOfShortAddress = value[13:2*NumOfAssociatedDevice]
if status == "0":
ret['success'] = "Command succesfully sent! The IEEE address is: %s" % (IEEEAddrRemoteDev)
self.write(tornado.escape.json_encode(ret))
elif status == "80":
ret['success'] = "Invalid Request Type"
self.write(tornado.escape.json_encode(ret))
else:
ret['error'] = "Device Not Found"
self.write(tornado.escape.json_encode(ret))
The error I receive in the developer consolle is this:
Uncaught TypeError: Cannot read property 'error' of null
$.ajax.successwsn.js:26 jQuery.Callbacks.firejquery-1.7.2.js:1075
jQuery.Callbacks.self.fireWithjquery-1.7.2.js:1193
donejquery-1.7.2.js:7538 jQuery.ajaxTransport.send.callback
in the js function. Why the function doesn't know 'error' or 'success'? Where is the problem???? I think the program don't enter never in the handler, but is blocked after, just in the js function.
Thank you very much for the help! It's a long post but it's simple to read! Please!
The quick read is the function isn't decorated correctly.
The line
response = yield tornado.gen.Task(http_client.fetch, como_url)
Means that you should declare the function thusly:
#tornado.web.asynchronous
#tornado.gen.engine
#tornado.web.authenticated
def get(self):
....
Note the addition of the two additional decorators.
I have a simple program which reads a large file containing few million rows, parses each row (numpy array) and converts into an array of doubles (python array) and later writes into an hdf5 file. I repeat this loop for multiple days. After reading each file, i delete all the objects and call garbage collector. When I run the program, First day is parsed without any error but on the second day i get MemoryError. I monitored the memory usage of my program, during first day of parsing, memory usage is around 1.5 GB. When the first day parsing is finished, memory usage goes down to 50 MB. Now when 2nd day starts and i try to read the lines from the file I get MemoryError. Following is the output of the program.
source file extracted at C:\rfadump\au\2012.08.07.txt
parsing started
current time: 2012-09-16 22:40:16.829000
500000 lines parsed
1000000 lines parsed
1500000 lines parsed
2000000 lines parsed
2500000 lines parsed
3000000 lines parsed
3500000 lines parsed
4000000 lines parsed
4500000 lines parsed
5000000 lines parsed
parsing done.
end time is 2012-09-16 23:34:19.931000
total time elapsed 0:54:03.102000
repacking file
done
> s:\users\aaj\projects\pythonhf\rfadumptohdf.py(132)generateFiles()
-> while single_date <= self.end_date:
(Pdb) c
*** 2012-08-08 ***
source file extracted at C:\rfadump\au\2012.08.08.txt
cought an exception while generating file for day 2012-08-08.
Traceback (most recent call last):
File "rfaDumpToHDF.py", line 175, in generateFile
lines = self.rawfile.read().split('|\n')
MemoryError
I am very sure that windows system task manager shows the memory usage as 50 MB for this process. It looks like the garbage collector or memory manager for Python is not calculating the free memory correcly. There should be lot of free memory but it thinks there is not enough.
Any idea?
EDIT
Adding my code here
I will put parts of my code. I am new to python, please pardon my python coding style.
module 1
def generateFile(self, current_date):
try:
print "*** %s ***" % current_date.strftime("%Y-%m-%d")
weekday=current_date.weekday()
if weekday >= 5:
print "skipping weekend"
return
self.taqdb = taqDB(self.index, self.offset)
cache_filename = os.path.join(self.cache_dir,current_date.strftime("%Y.%m.%d.h5"))
outputFile = config.hdf5.filePath(self.index, date=current_date)
print "cache file: ", cache_filename
print "output file: ", outputFile
tempdir = "C:\\rfadump\\"+self.region+"\\"
input_filename = tempdir + filename
print "source file extracted at %s " % input_filename
## universe
reader = rfaTextToTAQ.rfaTextToTAQ(self.tickobj) ## PARSER
count = 0
self.rawfile = open(input_filename, 'r')
lines = self.rawfile.read().split('|\n')
total_lines = len(lines)
self.rawfile.close()
del self.rawfile
print "parsing started"
start_time = dt.datetime.now()
print "current time: %s" % start_time
#while(len(lines) > 0):
while(count < total_lines):
#line = lines.pop(0) ## This slows down processing
result = reader.parseline(lines[count]+"|")
count += 1
if(count % 500000 == 0):
print "%d lines parsed" %(count)
if(result == None):
continue
ric, timestamp, quotes, trades, levelsUpdated, tradeupdate = result
if(len(levelsUpdated) == 0 and tradeupdate == False):
continue
self.taqdb.insert(result)
## write to hdf5 TODO
writer = h5Writer.h5Writer(cache_filename, self.tickobj)
writer.write(self.taqdb.groups)
writer.close()
del lines
del self.taqdb, self.tickobj
##########################################################
print "parsing done."
end_time = dt.datetime.now()
print "end time is %s" % end_time
print "total time elapsed %s" % (end_time - start_time)
defragger = hdf.HDF5Defragmenter()
defragger.Defrag(cache_filename,outputFile)
del defragger
print "done"
gc.collect(2)
except:
print "cought an exception while generating file for day %s." % current_date.strftime("%Y-%m-%d")
tb = traceback.format_exc()
print tb
module 2 - taqdb - to store parsed data in an array
class taqDB:
def __init__(self, index, offset):
self.index = index
self.tickcfg = config.hdf5.getTickConfig(index)
self.offset = offset
self.groups = {}
def getGroup(self,ric):
if (self.groups.has_key(ric) == False):
self.groups[ric] = {}
return self.groups[ric]
def getOrderbookArray(self, ric, group):
datasetname = orderBookName
prodtype = self.tickcfg.getProdType(ric)
if(prodtype == ProdType.INDEX):
return
orderbookArrayShape = self.tickcfg.getOrderBookArrayShape(prodtype)
if(group.has_key(datasetname) == False):
group[datasetname] = array.array("d")
orderbookArray = self.tickcfg.getOrderBookArray(prodtype)
return orderbookArray
else:
orderbookArray = group[datasetname]
if(len(orderbookArray) == 0):
return self.tickcfg.getOrderBookArray(prodtype)
lastOrderbook = orderbookArray[-orderbookArrayShape[1]:]
return np.array([lastOrderbook])
def addToDataset(self, group, datasetname, timestamp, arr):
if(group.has_key(datasetname) == False):
group[datasetname] = array.array("d")
arr[0,0]=timestamp
a1 = group[datasetname]
a1.extend(arr[0])
def addToOrderBook(self, group, timestamp, arr):
self.addToDataset(self, group, orderBookName, timestamp, arr)
def insert(self, data):
ric, timestamp, quotes, trades, levelsUpdated, tradeupdate = data
delta = dt.timedelta(hours=timestamp.hour,minutes=timestamp.minute, seconds=timestamp.second, microseconds=(timestamp.microsecond/1000))
timestamp = float(str(delta.seconds)+'.'+str(delta.microseconds)) + self.offset
## write to array
group = self.getGroup(ric)
orderbookUpdate = False
orderbookArray = self.getOrderbookArray(ric, group)
nonzero = quotes.nonzero()
orderbookArray[nonzero] = quotes[nonzero]
if(np.any(nonzero)):
self.addToDataset(group, orderBookName, timestamp, orderbookArray)
if(tradeupdate == True):
self.addToDataset(group, tradeName, timestamp, trades)
Module 3- Parser
class rfaTextToTAQ:
"""RFA Raw dump file reader. Readers single line (record) and returns an array or array of fid value pairs."""
def __init__(self,tickconfig):
self.tickconfig = tickconfig
self.token = ''
self.state = ReadState.SEQ_NUM
self.fvstate = fvstate.FID
self.quotes = np.array([]) # read from tickconfig
self.trades = np.array([]) # read from tickconfig
self.prodtype = ProdType.STOCK
self.allquotes = {}
self.alltrades = {}
self.acvol = 0
self.levelsUpdated = []
self.quoteUpdate = False
self.tradeUpdate = False
self.depth = 0
def updateLevel(self, index):
if(self.levelsUpdated.__contains__(index) == False):
self.levelsUpdated.append(index)
def updateQuote(self, fidindex, field):
self.value = float(self.value)
if(self.depth == 1):
index = fidindex[0]+(len(self.tickconfig.stkQuotes)*(self.depth - 1))
self.quotes[index[0]][fidindex[1][0]] = self.value
self.updateLevel(index[0])
else:
self.quotes[fidindex] = self.value
self.updateLevel(fidindex[0][0])
self.quoteUpdate = True
def updateTrade(self, fidindex, field):
#self.value = float(self.value)
if(self.tickconfig.tradeUpdate(self.depth) == False):
return
newacvol = float(self.value)
if(field == acvol):
if(self.value > self.acvol):
tradesize = newacvol - self.acvol
self.acvol = newacvol
self.trades[fidindex] = tradesize
if(self.trades.__contains__(0) == False):
self.tradeUpdate = True
else:
self.trades[fidindex] = self.value
if(not (self.trades[0,1]==0 or self.trades[0,2]==0)):
self.tradeUpdate = True
def updateResult(self):
field = ''
valid, field = field_dict.FIDToField(int(self.fid), field)
if(valid == False):
return
if(self.value == '0'):
return
if(self.prodtype == ProdType.STOCK):
fidindex = np.where(self.tickconfig.stkQuotes == field)
if(len(fidindex[0]) == 0):
fidindex = np.where(self.tickconfig.stkTrades == field)
if(len(fidindex[0]) == 0):
return
else:
self.updateTrade(fidindex, field)
else:
self.updateQuote(fidindex, field)
else:
fidindex = np.where(self.tickconfig.futQuotes == field)
if(len(fidindex[0]) == 0):
fidindex = np.where(self.tickconfig.futTrades == field)
if(len(fidindex[0]) == 0):
return
else:
self.updateTrade(fidindex, field)
else:
self.updateQuote(fidindex, field)
def getOrderBookTrade(self):
if (self.allquotes.has_key(self.ric) == False):
acvol = 0
self.allquotes[self.ric] = self.tickconfig.getOrderBookArray(self.prodtype)
trades = self.tickconfig.getTradesArray()
self.alltrades[self.ric] = [trades, acvol]
return self.allquotes[self.ric], self.alltrades[self.ric]
def parseline(self, line):
self.tradeUpdate = False
self.levelsUpdated = []
pos = 0
length = len(line)
self.state = ReadState.SEQ_NUM
self.fvstate = fvstate.FID
self.token = ''
ch = ''
while(pos < length):
prevChar = ch
ch = line[pos]
pos += 1
#SEQ_NUM
if(self.state == ReadState.SEQ_NUM):
if(ch != ','):
self.token += ch
else:
self.seq_num = int(self.token)
self.state = ReadState.TIMESTAMP
self.token = ''
# TIMESTAMP
elif(self.state == ReadState.TIMESTAMP):
if(ch == ' '):
self.token = ''
elif(ch != ','):
self.token += ch
else:
if(len(self.token) != 12):
print "Invalid timestamp format. %s. skipping line.\n", self.token
self.state = ReadState.SKIPLINE
else:
self.timestamp = datetime.strptime(self.token,'%H:%M:%S.%f')
self.state = ReadState.RIC
self.token = ''
# RIC
elif(self.state == ReadState.RIC):
if(ch != ','):
self.token += ch
else:
self.ric = self.token
self.token = ''
self.ric, self.depth = self.tickconfig.replaceRic(self.ric)
self.prodtype = self.tickconfig.getProdType(self.ric)
if(self.tickconfig.subscribed(self.ric)):
self.state = ReadState.UPDATE_TYPE
self.quotes, trades = self.getOrderBookTrade()
self.trades = trades[0]
self.acvol = trades[1]
else:
self.state = ReadState.SKIPLINE
# UPDATE_TYPE
elif(self.state == ReadState.UPDATE_TYPE):
if(ch != '|'):
self.token += ch
else:
self.update_type = self.token
self.token = ''
self.state = ReadState.FVPAIRS
#SKIPLINE
elif(self.state == ReadState.SKIPLINE):
return None
# FV PAIRS
elif(self.state == ReadState.FVPAIRS):
# FID
if(self.fvstate == fvstate.FID):
if(ch != ','):
if(ch.isdigit() == False):
self.token = self.value+ch
self.fvstate = fvstate.FIDVALUE
self.state = ReadState.FVPAIRS
else:
self.token += ch
else:
self.fid = self.token
self.token = ''
self.fvstate = fvstate.FIDVALUE
self.state = ReadState.FVPAIRS
# FIDVALUE
elif(self.fvstate == fvstate.FIDVALUE):
if(ch != '|'):
self.token += ch
else:
self.value = self.token
self.token = ''
self.state = ReadState.FVPAIRS
self.fvstate = fvstate.FID
# TODO set value
self.updateResult()
return self.ric, self.timestamp, self.quotes, self.trades, self.levelsUpdated, self.tradeUpdate
Thanks.
The only reliable way to free memory is to terminate the process.
So, if your main program spawns a worker process to do most of the work (the stuff that is done in one day) then when that worker process completes, the memory used will be freed:
import multiprocessing as mp
def work(date):
# Do most of the memory-intensive work here
...
while single_date <= self.end_date:
proc = mp.Process(target = work, args = (single_date,))
proc.start()
proc.join()