Develop Reference

matplotlib xlim TypeError: '>' not supported between instances of 'int' and 'list'

this is the original repo i'm trying to run in my computer: https://github.com/kreamkorokke/cs244-final-project
import os
import matplotlib.pyplot as plt
import argparse
from attacker import check_attack_type
IMG_DIR = "./plots"
def read_lines(f, d):
lines = f.readlines()[:-1]
for line in lines:
typ, time, num = line.split(',')
if typ == 'seq':
d['seq']['time'].append(float(time))
d['seq']['num'].append(float(num))
elif typ == 'ack':
d['ack']['time'].append(float(time))
d['ack']['num'].append(float(num))
else:
raise "Unknown type read while parsing log file: %s" % typ
def main():
parser = argparse.ArgumentParser(description="Plot script for plotting sequence numbers.")
parser.add_argument('--save', dest='save_imgs', action='store_true',
help="Set this to true to save images under specified output directory.")
parser.add_argument('--attack', dest='attack',
nargs='?', const="", type=check_attack_type,
help="Attack name (used in plot names).")
parser.add_argument('--output', dest='output_dir', default=IMG_DIR,
help="Directory to store plots.")
args = parser.parse_args()
save_imgs = args.save_imgs
output_dir = args.output_dir
attack_name = args.attack
if save_imgs and attack_name not in ['div', 'dup', 'opt'] :
print("Attack name needed for saving plot figures.")
return
normal_log = {'seq':{'time':[], 'num':[]}, 'ack':{'time':[], 'num':[]}}
attack_log = {'seq':{'time':[], 'num':[]}, 'ack':{'time':[], 'num':[]}}
normal_f = open('log.txt', 'r')
attack_f = open('%s_attack_log.txt' % attack_name, 'r')
read_lines(normal_f, normal_log)
read_lines(attack_f, attack_log)
if attack_name == 'div':
attack_desc = 'ACK Division'
elif attack_name == 'dup':
attack_desc = 'DupACK Spoofing'
elif attack_name == 'opt':
attack_desc = 'Optimistic ACKing'
else:
raise 'Unknown attack type: %s' % attack_name
norm_seq_time, norm_seq_num = normal_log['seq']['time'], normal_log['seq']['num']
norm_ack_time, norm_ack_num = normal_log['ack']['time'], normal_log['ack']['num']
atck_seq_time, atck_seq_num = attack_log['seq']['time'], attack_log['seq']['num']
atck_ack_time, atck_ack_num = attack_log['ack']['time'], attack_log['ack']['num']
plt.plot(norm_seq_time, norm_seq_num, 'b^', label='Regular TCP Data Segments')
plt.plot(norm_ack_time, norm_ack_num, 'bx', label='Regular TCP ACKs')
plt.plot(atck_seq_time, atck_seq_num, 'rs', label='%s Attack Data Segments' % attack_desc)
plt.plot(atck_ack_time, atck_ack_num, 'r+', label='%s Attack ACKs' % attack_desc)
plt.legend(loc='upper left')
x = max(max(norm_seq_time, norm_ack_time),max(atck_seq_time, atck_ack_time))
y = max(max(norm_seq_num, norm_ack_num),max(atck_seq_num, atck_ack_num))
plt.xlim(0, x)
plt.ylim(0,y)
plt.xlabel('Time (s)')
plt.ylabel('Sequence Number (Bytes)')
if save_imgs:
# Save images to figure/
if not os.path.exists(output_dir):
os.makedirs(output_dir)
plt.savefig(output_dir + "/" + attack_name)
else:
plt.show()
normal_f.close()
attack_f.close()
if __name__ == "__main__":
main()
after running this i get this error
Traceback (most recent call last):
File "plot.py", line 85, in <module>
main()
File "plot.py", line 66, in main
plt.xlim(0, a)
File "/usr/lib/python3/dist-packages/matplotlib/pyplot.py", line 1427, in xlim
ret = ax.set_xlim(*args, **kwargs)
File "/usr/lib/python3/dist-packages/matplotlib/axes/_base.py", line 3267, in set_xlim
reverse = left > right
TypeError: '>' not supported between instances of 'int' and 'list'
Done! Please check ./plots for all generated plots.
how can i solve this problem? or better yet if there is another way of running this project? i installed matplotlib via pip3 install matplotlib command (same with scapy) and my main python version is python2 right now but i run the project with python3, could the issue be about this? what am i missing? or is it about mininet itself?

The problem is in this line
x = max(max(norm_seq_time, norm_ack_time),max(atck_seq_time, atck_ack_time))
IIUC, you wanna assign to x the maximum value among all those four lists. However, when you pass two lists to the max function, such as max(norm_seq_time, norm_ack_time), it will return the list it considers the greater one, and not the highest value considering both lists.
Instead, you can do something like:
x = max(norm_seq_time + norm_ack_time + atck_seq_time + atck_ack_time)
This will concatenate the four lists into a single one. Then, the function will return the highest value among all of them. You might wanna do that to the calculation of y as well.
If this is not what you wanted, or if you have any further issues, please let us know.

with the help of a friend we solved this problem by changing a part in code into this:
max_norm_seq_time = max(norm_seq_time) if len(norm_seq_time) > 0 else 0
max_norm_ack_time = max(norm_ack_time) if len(norm_ack_time) > 0 else 0
max_atck_seq_time = max(atck_seq_time) if len(atck_seq_time) > 0 else 0
max_atck_ack_time = max(atck_ack_time) if len(atck_ack_time) > 0 else 0
x = max((max_norm_seq_time, max_norm_ack_time,\
max_atck_seq_time, max_atck_ack_time))
plt.xlim([0,x])
max_norm_seq_num = max(norm_seq_num) if len(norm_seq_num) > 0 else 0
max_norm_ack_num = max(norm_ack_num) if len(norm_ack_num) > 0 else 0
max_atck_seq_num = max(atck_seq_num) if len(atck_seq_num) > 0 else 0
max_atck_ack_num = max(atck_ack_num) if len(atck_ack_num) > 0 else 0
plt.ylim([0, max((max_norm_seq_num, max_norm_ack_num,\
max_atck_seq_num, max_atck_ack_num))])
```
writing here just in case anyone else needs it.

Python: assign a list to a function

Is there a way to assign a list based on a variable env and pass the result to function? I am passing a variable called env, which could be UPE, DEV, PRD for example. Based on the result, I want to assign the list respectively to the functions below. What would be the best approach?
UPE=['SERVER1','SERVER2','SERVER3','SERVER4']
DEV=['ServerA','ServerB']
PRD=['SERVER1','SERVER2','SERVER3','SERVER4']
if os.path.isfile('/myfile/' + configFile):
config_parser = ConfigParser()
config_parser.read('/myfile/' + configFile)
if actions == "start":
startOVD('start',UPE[3]) //I want pass the result of env setup variable
#ans = raw_input("Would you like to start OVD, MSAS,OAG : y | n : ")
if env == 'UPE':
startMSAS('start',UPE[0])
startOAG('start',UPE[1])
startOHS('start',UPE[2])
for section_name in sorted(config_parser.sections(), reverse=True):
adminURL = config_parser.get(section_name, 'ADMIN_URL')
adminUsername = config_parser.get(section_name, 'ADMIN_USER')
adminPassword = config_parser.get(section_name, 'ADMIN_PASS')
adminHost = config_parser.get(section_name, 'NM_HOST')
domainName = config_parser.get(section_name, 'DOMAIN_NAME')
domainDir = config_parser.get(section_name, 'DOMAIN_DIR')
admPort = adminURL[-4:]
printHeader('Initiating Starting Sequence')
startAdmin('start', adminHost, domainDir, domainName, admPort)
showServerStatus('start', adminUsername, adminPassword, adminURL)
if actions == "stop":
for section_name in (config_parser.sections()):
adminURL = config_parser.get(section_name, 'ADMIN_URL')
adminUsername = config_parser.get(section_name, 'ADMIN_USER')
adminPassword = config_parser.get(section_name, 'ADMIN_PASS')
adminHost = config_parser.get(section_name, 'NM_HOST')
domainName = config_parser.get(section_name, 'DOMAIN_NAME')
domainDir = config_parser.get(section_name, 'DOMAIN_DIR')
admPort = adminURL[-4:]
printHeader('Initiating Stopping Sequence')
showServerStatus('stop', adminUsername, adminPassword, adminURL)
stopAdmin(adminHost, domainDir, domainName, admPort)
if env == 'UPE':
stopMSAS('stop',UPE[0])
stopOAG('stop',UPE[1])
stopOHS('stop',UPE[2])
stopOVD('stop',UPE[3])

I would organize this by setting up a list of callbacks.
from functools import partial
start_funcs = [partial(startOVD 'start',UPE[3])
if env == 'UPE':
start_funcs.extend([partial(startMSAS, 'start', UPE[0]),
partial(startOAG, 'start', UPE[1])])
start_funcs.append(partial(startOHS, 'start', UPE[2]))
Add similar logic for the cases when env has a different value. In the end, you'll just iterate over start_funcs and call each function in order.
if actions == "start":
for f in start_funcs:
f()
# ...

Reading Serial Port data and saving to .csv Python

I'm having success reading in and printing some Serial Port data in a python script. I'm having difficulty with what tools/functions to use to change the Serial port data into a table or continuously append to a .csv. So after spending many hours researching everything about Serial Ports and python on stack overflow and other sites I'm reaching out for a hand as I don't know what steps to take next and I can't find information that helps answer my question.
Here is my script so far:
ser = serial.Serial('com4',
baudrate=38400,
timeout=1,
parity=serial.PARITY_NONE,
stopbits=serial.STOPBITS_ONE,
bytesize=serial.EIGHTBITS)
while True:
rawdata = (ser.readline().decode('ascii'))
print(rawdata)
Ok....great we have the Sensor data printing in the console looking like this:
$PDLMA,+11.91,+20.7,-0.1,-0.4*6C
$PDLMH,134829.844,+2644.8,+81.46,+3094.7,+21.99*6F
$PDLM1,134829.844,+1824.1,+127.69,+3276.7,+36.82*26
Now here's how the data from the sensor is structured:
$PDLMH 134829.844 2644.8 81.46 3094.7 21.99 *6F
Hhmmss.sss HCP05 HCPI PRP05 PRP05I Checksum
$PDLM1 134829.844 1824.1 127.69 3727.7 36.82 *26
Hhmmss.sss HCP10 HCP10I PRP10 PRP10I Checksum
$PDLMA 11.91 20.7 -0.1 -0.4
Voltage Temperature Pitch Roll
Now my goal is to get the final output to look like this:
Time HCP05 HCP05I PRP05 PRP05I HCP10 HCP10I PRP10 PRP10I Voltage Temp Pitch
(now) data data data data data data data data Data Data Data
So what should I do to take the serial data and transform it to what I need above as well as continuously reading in the data? I have tried dataframes, lists, etc but clearly I'm missing something here. In the .csv output file I would expect there to be about 120000 lines of data on average in our use case.
The Newbie of the day.

I achieved my desired output with the following code:
while dser.inWaiting() == 0:
pass
NMEA1 = dser.readline().decode("ascii")
while dser.inWaiting() == 0:
pass
NMEA2 = dser.readline().decode("ascii")
while dser.inWaiting() == 0:
pass
NMEA3 = dser.readline().decode("ascii")
while dser.inWaiting() == 0:
pass
NMEA4 = dser.readline().decode("ascii")
while dser.inWaiting() == 0:
pass
NMEA1_array = NMEA1.split(',')
NMEA2_array = NMEA2.split(',')
NMEA3_array = NMEA3.split(',')
NMEA4_array = NMEA4.split(',')
if NMEA1_array[0] == '$PDLMH':
HCP05 = NMEA1_array[2]
PRP05 = NMEA1_array[4]
if NMEA1_array[0] == '$PDLM1':
HCP10 = NMEA1_array[2]
PRP10 = NMEA1_array[4]
if NMEA1_array[0] == '$PDLM2':
HCP20 = NMEA1_array[2]
PRP20 = NMEA1_array[4]
if NMEA1_array[0] == '$PDLMA':
Voltage = NMEA1_array[1]
Temperature = NMEA1_array[2]
Pitch = NMEA1_array[3]
Roll = NMEA1_array[4].partition("*")[0]
if NMEA2_array[0] == '$PDLMH':
HCP05 = NMEA2_array[2]
PRP05 = NMEA2_array[4]
if NMEA2_array[0] == '$PDLM1':
HCP10 = NMEA2_array[2]
PRP10 = NMEA2_array[4]
if NMEA2_array[0] == '$PDLM2':
HCP20 = NMEA2_array[2]
PRP20 = NMEA2_array[4]
if NMEA2_array[0] == '$PDLMA':
Voltage = NMEA2_array[1]
Temperature = NMEA2_array[2]
Pitch = NMEA2_array[3]
Roll = NMEA2_array[4].partition("*")[0]
if NMEA3_array[0] == '$PDLMH':
HCP05 = NMEA3_array[2]
PRP05 = NMEA3_array[4]
if NMEA3_array[0] == '$PDLM1':
HCP10 = NMEA3_array[2]
PRP10 = NMEA3_array[4]
if NMEA3_array[0] == '$PDLM2':
HCP20 = NMEA3_array[2]
PRP20 = NMEA3_array[4]
if NMEA3_array[0] == '$PDLMA':
Voltage = NMEA3_array[1]
Temperature = NMEA3_array[2]
Pitch = NMEA3_array[3]
Roll = NMEA3_array[4].partition("*")[0]
if NMEA4_array[0] == '$PDLMH':
HCP05 = NMEA4_array[2]
PRP05 = NMEA4_array[4]
if NMEA4_array[0] == '$PDLM1':
HCP10 = NMEA4_array[2]
PRP10 = NMEA4_array[4]
if NMEA4_array[0] == '$PDLM2':
HCP20 = NMEA4_array[2]
PRP20 = NMEA4_array[4]
if NMEA4_array[0] == '$PDLMA':
Voltage = NMEA4_array[1]
Temperature = NMEA4_array[2]
Pitch = NMEA4_array[3]
Roll = NMEA4_array[4].partition("*")[0]
if () is not None:
return (float(HCP05), float(PRP05), float(HCP10),
float(PRP10), float(HCP20), float(PRP20),
float(Voltage), float(Temperature), float(Pitch),
float(Roll))
Not to sure if this is the best result but it is now working.

Python unified diff with line numbers from both "files"

I'm trying to figure out a way to create unified diffs with line numbers only showing N lines of context. I have been unable to do this with difflib.unified_diff. I need to show changes in both files.
The closest I can come is using diff on the command line like so:
/usr/bin/diff
--unchanged-line-format=' %.2dn %L'
--old-line-format="-%.2dn %L"
--new-line-format="+%.2dn %L"
file1.py
file2.py
BUT I only want to show N lines of context, and /usr/bin/diff doesn't seem to support context with a custom line format (eg. -U2 is not compatible with --line-format "conflicting output style options").
Below is an example of what I'd like to accomplish (the same output as the above diff, but only showing 1 line of context surrounding changes):
+01: def renamed_function()
-01: def original_function():
02:
+03: """ Neat stuff here """
04:
21:
+22: # Here's a new comment
23:
85: # Output the value of foo()
+86: print "Foo is %s"%(foo())
-86: print foo()
87:

I was able to figure out something very close to what I wanted to do. It's slower than regular diff, though. Here's the entire code, from my project GitGate.
def unified_diff(to_file_path, from_file_path, context=1):
""" Returns a list of differences between two files based
on some context. This is probably over-complicated. """
pat_diff = re.compile(r'## (.[0-9]+\,[0-9]+) (.[0-9]+,[0-9]+) ##')
from_lines = []
if os.path.exists(from_file_path):
from_fh = open(from_file_path,'r')
from_lines = from_fh.readlines()
from_fh.close()
to_lines = []
if os.path.exists(to_file_path):
to_fh = open(to_file_path,'r')
to_lines = to_fh.readlines()
to_fh.close()
diff_lines = []
lines = difflib.unified_diff(to_lines, from_lines, n=context)
for line in lines:
if line.startswith('--') or line.startswith('++'):
continue
m = pat_diff.match(line)
if m:
left = m.group(1)
right = m.group(2)
lstart = left.split(',')[0][1:]
rstart = right.split(',')[0][1:]
diff_lines.append("## %s %s ##\n"%(left, right))
to_lnum = int(lstart)
from_lnum = int(rstart)
continue
code = line[0]
lnum = from_lnum
if code == '-':
lnum = to_lnum
diff_lines.append("%s%.4d: %s"%(code, lnum, line[1:]))
if code == '-':
to_lnum += 1
elif code == '+':
from_lnum += 1
else:
to_lnum += 1
from_lnum += 1
return diff_lines

Python (3) readline returns empty string regardless of what is in the file

I have a Python (3) program that reads some data in from a file, gets the relevant parts and spits it out to various other places. The problem is that file.readline() has started refusing to get any lines from any file. It just returns an empty string as if it was at the end of the file. The files are not the problem, I checked that with a quick script (read file, print line by line). The possible solution (although I don't know why) is I used 2to3 to convert the code to python3, but looking at the diff it printed I don't see any modifications to the file I/O stuff.
Here is the program:
import threading
import time
def runFile(eventFile, channelRefFileName, serialPorts, timeScale, maxPEs):
timer0 = threading.Timer(.0001, turnOffLED, [0, 0, 0, 1])
numEvents = getNumEvents(eventFile)
eventCount = 0
while 1:
linein = eventFile.readline()
#if linein == '' or '\n' or '\r\n' or '\r': break
#above checks if the end of file has been reached, below is debugging
if linein == '': print('EOF')
if linein == '\n': print('linefeed')
if linein == '\r\n': print('carrige return/ linefeed')
if linein == '\r' : print('carrige return')
if linein == 'New Event\n':
print('New Event', eventCount, ': file is',
(eventCount/numEvents)*100, '% done.')
eventCount += 1
#insert event sleep time here
continue
linein = linein.split(' ')
del(linein[::2])
linein[2] = linein[2][:-1]
linein[1] = float(linein[1])
if linein[1] < 0: linein[1] = linein[1] * (-1)
channel = channelLookup(linein[0], channelRefFileName)
#serialPorts[channel[0]].write(str(channel[1],linein[2]))
if timer0.isAlive:
#use backup timer
timer1 = threading.Timer(0.005, turnOffLED, [serialPorts, channel[0], channel[1], 0])
timer1.start()
else:
#use primary timer
timer0 = threading.Timer(0.005, turnOffLED, [serialPorts, channel[0], channel[1], 0])
timer0.start()
print('Sleeping:', linein[1]*timeScale)
time.sleep(linein[1]*timeScale)
def channelLookup(channel, channelRefFile):
channelRef = open(channelRefFile, 'r')
for line in channelRef:
if channel in line:
linein = line
break
linein = linein.split('-')
channelRef.close()
return linein[1:]
def turnOffLED(serialPorts, arduino, pmt, bs):
if bs: return -1
#serialPorts[arduino].write(str(pmt,0))
def getNumEvents(eventFile):
i = 0
for lines in eventFile:
if lines == 'New Event\n':
i += 1
return i
The above is run by:
import hawc
import datetime
timeScale = 0.001
#timeScale of 0.000001 results in runtime of 0:06:52.858136
#a timeScale of 0.001 is 9:28:34.837992
eventSleep = 0
maxPEs = 1000
serialPorts = [0, 1, 2, 3, 4]
channelRefFileName = 'Data/hawc-arduino_channel_lookup'
st = datetime.datetime.now()
print('Start time is', st)
eventFile = open('Data/events.txt', 'r')
hawc.runFile(eventFile, channelRefFileName, serialPorts, timeScale, maxPEs)
eventFile.close()
et = datetime.datetime.now()
print('End time is', et)
print('File has run for', et-st)
print('--------Done---------')
And here is some sample data that the program might handle (one file might be about 500000 lines):
Channel: 447 Time: 121.263 PEs: 2263.38
Channel: 445 Time: 118.556 PEs: 1176.54
Channel: 448 Time: 120.384 PEs: 1159.59
Channel: 446 Time: 122.798 PEs: 983.949
Channel: 499 Time: 129.983 PEs: 762.07

getNumEvents consumes the entire file, so when you go around reading it, its pointer is already at the end. Instead, reset the file pointer with seek, like this:
def runFile(eventFile, channelRefFileName, serialPorts, timeScale, maxPEs):
timer0 = threading.Timer(.0001, turnOffLED, [0, 0, 0, 1])
numEvents = getNumEvents(eventFile)
eventFile.seek(0, 0)
eventCount = 0
for linein in eventFile:
if linein == '': print('EOF')
....