I have a program that interfaces with another program to process data. The program I have written has three main parts (separated by lines of ### below). When I run the entire program, it does not work (specifically the second subprocess call doesn't execute and produce a summary file for which the third part of the program is responsible for dealing with). However, whenever I break them into three separate programs and run them in the terminal, everything is fine. Any tips or suggestions as to how to get this to run as one program?
#### part 1:
import glob
import subprocess
import os
import datetime
import matplotlib.pyplot as plt
import csv
import re
import ntpath
x = open('data.txt', 'w')
m = open('graphing_data.txt', 'w')
ckopuspath= '/Volumes/DAVIS/sfit-ckopus/ckopus'
command_init = 'sfit4Layer0.py -bv5 -fh'
subprocess.call(command_init.split(), shell=False)
for line in open('sfit4.ctl', 'r'): # write in later specific directory line
p = line.strip()
if p.startswith('band.1.nu_start'):
a,b = p.split('=')
b = float(b)
b = "{0:.3f}".format(b)
lowwav = b
lowwav = float(lowwav)
if p.startswith('band.1.nu_stop'):
a,b = p.split('=')
b = float(b)
b = "{0:.3f}".format(b)
highwav = b
highwav = float(highwav)
with open('/Volumes/DAVIS/calpy_em27_neu/spectra_out_demo/info.txt', 'rt') as infofile: # the info.txt file created by CALPY
for count, line in enumerate(infofile):
with open('t15asc.4', 'w') as t:
lat = re.search('Latitude of location:\s*([^;]+)', line, re.IGNORECASE).group(0)
lat = lat.split()
lat = lat[3]
lat = float(lat)
lon = re.search('Longitude of location:\s*([^;]+)', line, re.IGNORECASE).group(0)
lon = lon.split()
lon = lon[3]
lon = float(lon)
date = re.search('Time of measurement \(UTC\): ([^;]+)', line).group(0)
date = date.split()
yeardate = date[4]
yeardate = yeardate.split('-')
year = int(yeardate[0])
month = int(yeardate[1])
day = int(yeardate[2])
time = date[5]
time = time.split(':')
hour = int(time[0])
minute = int(time[1])
second = time[2]
second = second.split('.')
second = int(second[0])
dur = re.search('Duration of measurement \[s\]: ([^;]+)', line).group(0)
dur = dur.split()
dur = float(dur[4])
numpoints = re.search('Number of values of one scan:\s*([^;]+)', line, re.IGNORECASE).group(0)
numpoints = numpoints.split()
numpoints = float(numpoints[6])
fov = re.search('semi FOV \[rad\] :\s*([^;]+)', line, re.IGNORECASE).group(0)
fov = fov.split()
fov = fov[3]
fov = float(fov[1:])
sza = re.search('sun Azimuth \[deg\]:\s*([^;]+)', line, re.IGNORECASE).group(0)
sza = sza.split()
sza = float(sza[3])
snr = 0.0000
roe = 6396.2
res = 0.5000
calpy_path = '/Volumes/DAVIS/calpy_em27_neu/spectra_out_demo/140803/*' # the CALPY output files!
files1 = glob.glob(calpy_path)
with open(files1[count], 'r') as g:
ints = []
ints_count = 0
for line in g:
wave_no, intensity = [float(item) for item in line.split()]
if lowwav <= wave_no <= highwav:
ints.append(str(intensity))
ints_count = ints_count + 1
spacebw = (highwav - lowwav)/ ints_count
d = datetime.datetime(year, month, day, hour, minute, second)
t.write('{:>12.5f}{:>12.5f}{:>12.5f}{:>12.5f}{:>12.5f}'.format(sza,roe,lat,lon,snr)) # line 1
t.write("\n")
t.write('{:>10d}{:>5d}{:>5d}{:>5d}{:>5d}{:>5d}'.format(year,month,day,hour,minute,second)) # line 2
t.write("\n")
t.write( ('{:%Y/%m/%d %H:%M:%S}'.format(d)) + "UT Solar Azimuth:" + ('{:>6.3f}'.format(sza)) + " Resolution:" + ('{:>6.4f}'.format(res)) + " Duration:" + ('{:>6.2f}'.format(dur))) # line 3
t.write("\n")
t.write('{:>22.12f}{:>22.12f}{:>22.12e}{:>15d}'.format(lowwav,highwav,spacebw,ints_count)) # line 4
t.write("\n")
t.write('\n'.join(map(str, ints)) + '\n')
######################################## part 2:
command_two = 'sfit4Layer0.py -bv5 -fs'
subprocess.call(command_two.split(), shell=False) #writes the summary file
######################################## part 3
infile = '/Volumes/DAVIS/calpy_em27_neu/spectra_out_demo/summary'
lines = open(infile, 'r').readlines()
line_number = 0
line4= lines[line_number + 3]
variables = line4.split(" ")
sample = variables[1]
time = variables[2]
Z = variables[3]
A = variables[4]
D = variables[5]
R = variables[6]
P = variables[7]
V = variables[8]
E = variables[9]
x.write('{0} {1} {2} {3} '.format(sample, time, Z, A))
time = time[:-2]
numofgas = lines[line_number + 5]
numofgas = int(numofgas.strip())
x.write(str(numofgas))
count2 = 0
while count2 < numofgas:
gasinfo = lines[line_number + 7 + count2]
data = gasinfo.split()
gasnum = data[0]
gasname = data[1]
IFPRF = data[2]
apprcol = data[3]
retcol = data[4]
x.write(' {0} {1}'.format(gasname, retcol))
m.write('{0},{1}\n'.format(time, retcol))
count2 = count2 + 1
line17 = lines[line_number + 10 + count2]
print(numofgas)
params = line17.split()
bandstart = params[1]
bandstop = params[2]
space = params[3]
nptsb = params[4]
pmax = params[5]
fovdia = params[6]
meansnr = params[7]
x.write(' {0} {1} {2} {3} '.format(bandstart, bandstop, nptsb, meansnr))
line21 = lines[line_number + 14 + count2]
info = line21.split()
fitrms = info[0]
chi2 = info[1]
dofsall = info[2]
dofstrg = info[3]
iter = info[4]
maxxiter = info[5]
x.write('{0} {1} {2} {3} {4}'.format(fitrms, chi2, dofsall, dofstrg, iter))
x.write('\n')
x.close()
m.close()
Related
def main_loop():
global errname, errtime, error_detail, conclusion
error_detail = ""
facts_all = {}
facts = []
buffer = 0
current_time = datetime.now()
while os.path.exists("C:\Winusr"):
print(paths["wintrace"])
try:
start_point = 0
old_size = os.path.getsize(paths["wintrace"])
while os.path.getsize(paths["wintrace"])>= old_size:
#fo = open(paths["wintrace"], "rb")
#fo.seek(start_point,1)
shutil.copyfile(paths["wintrace"], "C:\Winusr\wintrace1.log")
fo = open("C:\Winusr\wintrace1.log", "rb")
fo.seek(start_point, 1)
errtime = datetime(1900, 1, 1)
old_size = os.path.getsize(paths["wintrace"])
#start from here
for line in fo.readlines():
line = str(line.decode('ISO-8859-1'))
print(line)
if fnmatch.fnmatch(line, "*START DUMP LOG BUFFER*"):
buffer = 1
if fnmatch.fnmatch(line, "*END DUMP LOG BUFFER*"):
buffer = 0
if buffer == 1:
continue
facts_all = collect_facts(line,facts_all,key_string,key_value_dic)
for pattern in error_detect:
if fnmatch.fnmatch(line, pattern):
try:
err_type = df[df["Error Detect Keyword"] == pattern]["Err Type"].to_string(index=False).lstrip()
errname = df[df["Err Type"] == err_type]["Error Name"].tolist()[0].lstrip()
errtime = datetime.strptime(
datetime.fromtimestamp(os.path.getmtime(paths["wintrace"])).strftime("%Y-%m-%d") + " " + line[:8], "%Y-%m-%d %H:%M:%S") #"%d-%b-%Y %H:%M:%S"
#errtime = datetime.fromtimestamp(os.path.getmtime(paths["wintrace"])).strftime("%Y-%m-%d") + " " + line[:8]
#errtime = errtime.strftime('%Y-%m-%d %H:%M:%S')
product = re.findall(r"[/](.+?)[.]", paths["cur"])
product = product[0].split("/")[-1]
tester = tester_name(paths["cur"])
if len(facts_all) != 0:
facts.append(errname)
#idex = 9999
for fact, line in facts_all.items():
if fact in dic1[errname]:
error_detail = error_detail + line + '\n'
facts.append(fact)
print("err_detail1", error_detail)
if len(facts) != 1:
facts = list(set(facts))
conclusion = inference_engine(facts)
print("errtime", errtime)
print("current_time", current_time)
if conclusion != "cannot find solution for this error" and errtime > current_time:
solutions = sop2(errlist, errname, conclusion)
row = recording(tester, product, errname, errtime, error_detail, conclusion)
print("gg pop out GUI!!!")
#send_email(errname, errtime, tester, error_detail)
GUI(errname, errtime, error_detail, conclusion, solutions, row)
current_time = datetime.now()
workbook = xlrd.open_workbook(r"G:\expert system data\Machine Database.xls")
workbook1 = copy(workbook)
ws1 = workbook1.get_sheet(0)
style = xlwt.XFStyle()
style.num_format_str = 'yyyy-mm-dd hh:mm:ss'
ws1.write(row, 8, current_time, style)
workbook1.save(r"G:\expert system data\Machine Database.xls")
error_detail = ""
facts_all = {}
facts = []
error_detail = ""
facts_all = {}
facts = []
except:
continue
start_point = fo.tell()
fo.close()
except:
continue
else:
main_loop()
the paths["wintrace"] is ""C:\Winusr\Wintrace.log", i dont want it is open cause sometimes need to change its name or delete, i copy this file and open the copied one, but it still show it is open, can u help me check where it is opened? besides, i use "filepath = tkinter.filedialog.askopenfilename()", but dont think it will open the wintrace file.the error screenshot
I am trying to implement this CNN model for stock prize prediction: https://github.com/ZezhouLi/Convolutional-Networks-for-Stock-Predicting
But I am facing the issue while preprocessing the data for the implementation. The preprocess step is consuming a lot of RAM. (My system has 32 GB RAM and 256 GB SSD Hard disk)
Here is the file that is consuming the RAM and at last gives a Memory Error:
import numpy as np
import matplotlib.pyplot as plt
import glob
import math
from PIL import Image
import statsmodels.api as sm
def r_squared(y_true, y_hat):
ssr = 0
sst = 0
e = np.subtract(y_true, y_hat)
y_mean = np.mean(y_true)
for item in e:
ssr += item**2
for item in y_true:
sst += (item - y_mean)**2
r2 = 1 - ssr / sst
return r2
def data_process(data):
processed_data = []
for item in data:
m = np.mean(item)
s = np.std(item)
normal_item = [(float(i)-m)/s for i in item]
normal_item.insert(0, 1)
processed_data.append(normal_item)
return processed_data
def get_pixel_values():
file_name = r'\figures'
pixels = []
for filename in glob.glob(file_name + '\*.png'):
im = Image.open(filename)
temp_pixels = list(im.getdata())
pixels.append(temp_pixels)
return pixels
def find_returns(data):
returns = []
for group in data:
count = 30
while count <= (len(group)-5):
current_data = group[count-1]
future_data = group[count+4]
p1 = np.mean(current_data)
p2 = np.mean(future_data)
returns.append(math.log(p2/p1))
count += 1
return returns
def convert_image():
size = 54, 32
file_name = r'\figures'
for filename in glob.glob(file_name + '\*.png'):
img = Image.open(filename)
img.thumbnail(size)
img = img.convert('L')
img.save(filename)
def plot_data(data):
t = np.arange(0, 29, 1)
file_name_number = 0
fig = plt.figure(frameon=False)
for group in data:
count = 30
while count <= (len(group)-5):
high = []
low = []
for item in group[count-30:count]:
high.append(item[0])
low.append(item[1])
file_name = r'\fig_' + str(file_name_number)
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.set_axis_off()
fig.add_axes(ax)
ax.plot(t, high[0:-1], 'b', t, low[0:-1], 'g')
fig.savefig(r'\figures' + file_name)
fig.clf()
file_name_number += 1
count += 1
print('Created %d files!' % file_name_number)
def extract_useful_data(data):
groups = []
for group in data:
temp_buffer = []
for item in group:
temp = [item[2], item[3]]
temp = [float(i) for i in temp]
temp_buffer.append(temp)
groups.append(temp_buffer)
return groups
def split_data(data):
groups = []
for item in data:
temp_buffer = []
for string in item:
number = string.split(',')
temp_buffer.append(number)
groups.append(temp_buffer)
return groups
def extract_data():
file_name = r'\data.txt'
infile = open(file_name, 'r')
temp_buffer = []
for line in infile:
temp_buffer.append(line.strip('\n'))
temp_buffer = temp_buffer[8:]
i = 0
groups = []
temp = []
for item in temp_buffer:
if i != 390:
temp.append(item)
i += 1
else:
groups.append(temp)
temp = []
i = 0
groups.append(temp)
infile.close()
return groups
def main():
original_data = extract_data()
splitted_data = split_data(original_data)
useful_data = extract_useful_data(splitted_data)
plot_data(useful_data)
convert_image()
returns = np.asarray(find_returns(useful_data))
training_data = np.asarray(get_pixel_values())
training_data = sm.add_constant(training_data, has_constant='add')
results = sm.OLS(returns[0:4340], training_data[0:4340]).fit()
y_in_sample = results.predict(training_data[0:4340])
r2 = r_squared(returns[0:4340], y_in_sample)
print r2
if __name__ == "__main__":
main()
I have got Memory Error, which occurs when the program consumes all of the RAM memory of the system. Please improve on the program.
I have a python code to parse a 1 TB log file, but the problem is my result is shown after the parsing process is finished. So for that I need to wait for 12 hours, and after 12 hours only then the result is shown. I want to know how can I parse a log file and know the result of the parsing speed every 10 seconds.
This is my code:
import re
import timeit
log_file = '/Users/kiya/Desktop/mysql/ipscan/ip.txt'
output_file ='/Users/kiya/Desktop/mysql/ipscan/k2u.csv'
name_to_check = 'MBX_AUTHENTICATION_FAILED'
class Log_minning:
def __init__(self):
self.counter = 0
def get_userdata(self):
user_att = []
list_usr = []
counterr = 0
with open(log_file, encoding='utf-8') as infile:
for line in infile:
if name_to_check in line:
username = re.search(r'(?<=userName=)(.*)(?=,)', line)
username = username.group()
date = re.search(r"([12]\d{3}(0[1-9]|1[0-2])+"
"(0[1-9]|[12]\d|3[01]))", line)
date = date.group()
time = re.search(r"(\d{9}\+\d{4})", line)
time = time.group()
ip = re.search(
r'(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.)'
'{3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])',
line
)
ip = ip.group()
user_att.append(username)
user_att.append(date)
user_att.append(time)
user_att.append(ip)
list_usr.append(user_att)
counterr = counterr + 1
self.counter = counterr
return list_usr
if __name__ == "__main__":
lm = Log_minning()
the_time = timeit.Timer(lm.get_userdata).repeat(1, 1000)
sing_time = min(the_time)/1000
speed = 600 / sing_time * lm.counter
# for line in lm.get_userdata():
# print(line)
print(
"Processing " + str(lm.counter) + " in " + str(the_time) +
"\nThe speed aproximately " + str(speed) + " data in 10 sec"
)
This is the fully scanned seconds
Processing 117 in [6.646515152002394]
import os
import sys
import math
import cvxopt as cvx
import picos as pic
import pandas as pd
import matplotlib.pyplot as plt
from gurobipy import *
from statsmodels.tsa.arima_model import ARIMA
import numpy as np
from scipy import *
#import DeferableLoad
OPTmodel = Model('OPTIMIZER')
#general parameters
Tamb =22
N = 1440 # maximum iteration
i = range(1, N)
COP= 3.4 # Coeffient of performance
'''
Prediction need to be added here
'''
# Datacenter room defintion
R = 10 #length of room
B = 7
H = 9 #Height of room
L = 10
dT = 60
A = 2*((L*B)+(B*H)+(H*L))
Thick = 0.33 # thickness of wall
k = 0.7 # thermal conductivity of wall
mAir = 1.2 * (L * B * H)
C = 718
landa = k * A / Thick
a0 = 0.05 / dT
a1 = 1
ki = math.exp(-(landa * 60) / (mAir * C)) # value that constant and its related to property of room
kc = (1 - ki) * a0
ko = (1 - ki) * a1
kp = (1 - ki) * (COP / landa)
Tmin= 18
Tmax= 27
Tamb= 22
PcoolingRated = 100
Pbess_rated = 30.462
Pbess_ratedN = -30.462
Ebess_min = 0
Ebess_max = 300
with open ('Pcooling.csv','r') as f:
Pcooling = []
for line in f:
Pcooling.append(line)
f.close()
with open ('ITpower.csv','r') as f1:
ITload = []
for line1 in f1:
ITload.append(line1)
f1.close()
with open ('DR.csv','r') as f2:
DR =[]
for line2 in f2:
DR.append(line2)
f2.close()
print ITload
print Pcooling
print DR
for i in range(1,200):
for it in range(1, 1440):
Tm = np.empty(1440)
Tm.fill(18)
TmA = np.empty(1440)
TmA.fill(27)
Phvac_flex = {}
Phvac_up = {}
Phvac_down_= {}
Phvac_up_ = {}
Pbess_out_ = {}
Pbess_in_ = {}
Phvac_down = {}
Pbess_flex_ = {}
Pbess_flex = {}
Phvac_flex_ = {}
Pbess_in = {}
Pdc = {}
Pdc_base = {}
Pflex_i = {}
Tdc_i = {}
Pbess_out ={}
Ebess_i = {}
Phvac_flex[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS,name="PHVAC_flex"+str(i))
Phvac_up[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PHVAC_up" + str(i))
Phvac_up_[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PHVAC_up_" + str(i))
Phvac_down_[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PHVAC_down_" + str(i))
Pbess_out_[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PBESS_out_" + str(i))
Pbess_in_[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PBESS_in_" + str(i))
Phvac_down[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PHVAC_down" + str(i))
Pbess_flex_[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PBESS_flex_" + str(i))
Pbess_flex[i] = OPTmodel.addVar(lb=-GRB.INFINITY,ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PBESS_flex" + str(i))
Phvac_flex_[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PHVAC_flex_" + str(i))
Pbess_in[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PBESS_in" + str(i))
Pdc[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PDC" + str(i))
Pdc_base[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PDC_base" + str(i))
Pflex_i[i]= OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="Pflex_i" + str(i))
Tdc_i[i]= OPTmodel.addVar(ub=GRB.INFINITY,vtype = GRB.CONTINUOUS, name = "Tdc_i" + str(i))
Pbess_out[i] = OPTmodel.addVar(lb=-GRB.INFINITY,ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PBESS_out" + str(i))
Ebess_i[i]= OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS,name="Ebess_i" + str(i))
Pflex_i[1] = 0
Pflex_i[1] = 0
Tdc_i[0] = 18
Phvac_flex[1] = 0
# Phvac_flex_[1] = 0
Phvac_down[1] = 0
Phvac_up[1] = 0
Phvac_down_[1] = 0
Phvac_up_[1] = 0
# Phvac_down_pos[1] = 0
# Phvac_up_pos(1) = 0;
Pbess_flex[1] = 0
# Pbess_flex_[1] = 0
Pbess_out[1] = 0
Pbess_in[1] = 0
# Pbess_out_[1] = 0
Pbess_in_[1] = 0
# Pbess_out_pos[1] = -250
# Pbess_in_pos(1) = 250;
Ebess_i[1] = 150
OPTmodel.update()
'''
if float(DR[i]) > 0:
Phvac_down_[i] = 0
Phvac_up_[i] = float(DR[i])
Pbess_out_[i] = 0
Pbess_in_[i] = float(DR[i])
#Pbess_flex_[i] = Pbess_in_[i] + Pbess_out_[i]
#Phvac_flex_[i] = Phvac_down_[i] + Phvac_up_[i]
OPTmodel.update()
elif float(DR[i]) < 0:
Phvac_down_[i] = float(DR[i])
Phvac_up_[i] = 0
#Phvac_flex_[i] = Phvac_down_[i] + Phvac_up_[i]
Pbess_out_[i] = float(DR[i])
Pbess_in_[i] = 0
#Pbess_flex_[i] = Pbess_in_[i] + Pbess_out_[i]
OPTmodel.update()
else:
Phvac_down_[i] = 0
Phvac_up_[i] = 0
Phvac_flex_[i] = Phvac_down_[i] + Phvac_up_[i]
Pbess_out_[i] = 0
Pbess_in_[i] = 0
Pbess_flex_[i] = Pbess_in_[i] + Pbess_out_[i]
OPTmodel.update()
'''
#print Phvac_up.values()
#print Phvac_flex_[i]
print OPTmodel
OPTmodel.update()
ConHVAC1 = OPTmodel.addConstr(Phvac_flex[i] == Phvac_up[i] + Phvac_down[i], name='ConHVAC1')
ConHVAC2 = OPTmodel.addConstr(0 <= Phvac_flex[i] , name='ConHVAC2')
ConHVAC3 = OPTmodel.addConstr(Phvac_flex[i] <= PcoolingRated, name='ConHVAC3')
PH = pd.read_csv('Pcooling.csv')
PHVAC = PH.values
newList2 = map(lambda x: x / 1000, PHVAC)
p=[]
p=PcoolingRated-newList2[i]
#CONHVAC4 = OPTmodel.addConstr(Phvac_up[i]==np.minimum((Phvac_up_[i]),(float(newList2[i]))))
#Phvac_u(1:MaxIter) == min(Phvac_u_(1:MaxIter), (repelem(Phvac_max, MaxIter) - (Pcooling(1:MaxIter)'/1000)))
ConTemp1 = OPTmodel.addConstr(Tm[it] <= Tdc_i[i] <= TmA[it], name='ConTemp1')
ConBESS1 = OPTmodel.addConstr(Pbess_ratedN <= Pbess_flex[i] <= Pbess_rated, name='ConBESS1')
ConBESS2 = OPTmodel.addConstr(Pbess_flex[i] == Pbess_in[i] + Pbess_out[i], name='ConBESS2')
ConBESS3 = OPTmodel.addConstr(0 <= Pbess_in[i] <= min(Pbess_rated, Pbess_in_[i]), name='ConBESS3')
ConBESS4 = OPTmodel.addConstr(np.maximum(Pbess_ratedN,Pbess_out_[i]) <= Pbess_out[i]<=0 , name='ConBESS4') # need to modifty
ConEBESS1 = OPTmodel.addConstr(Ebess_min <= Ebess_i[i], name='ConEBESS1')
ConEBESS2 = OPTmodel.addConstr(Ebess_i[i] <= Ebess_max, name='ConEBESS2')
D = pd.read_csv('DR.csv').values
DRN = map(lambda x: x / 1000, D)
PDRN=map(lambda x: x / 4.8, DRN)
if float((PDRN[i])) > 0:
CON1 = OPTmodel.addConstr(Pbess_flex_[i] == Pbess_in_[i] + Pbess_out_[i],'CON1')
CON2 = OPTmodel.addConstr(Phvac_flex_[i] == Phvac_up_[i] + Phvac_down_[i],'CON2')
CON3=OPTmodel.addConstr(Phvac_down_[i] == 0, name='CON3')
CON4=OPTmodel.addConstr(Phvac_up_[i] == float((PDRN[i])),name='CON4')
CON5=OPTmodel.addConstr(Pbess_out_[i] == 0,name='CON5')
CON6=OPTmodel.addConstr(Pbess_in_[i] == float((PDRN[i])),name='CON6')
elif float(np.transpose(PDRN[i])) < 0:
CON7=OPTmodel.addConstr(Phvac_down_[i] == float(np.transpose(PDRN[i])),name='CON7')
CON8=OPTmodel.addConstr(Phvac_up_[i] == 0,name='CON8')
# Phvac_flex_[i] = Phvac_down_[i] + Phvac_up_[i]
CON9=OPTmodel.addConstr(Pbess_out_[i] == float((PDRN[i])),name='CON9')
CON10=OPTmodel.addConstr(Pbess_in_[i] == 0,name='CON10')
else:
CON11=OPTmodel.addConstr(Phvac_down_[i] == 0,name='CON11')
CON12=OPTmodel.addConstr(Phvac_up_[i] == 0,name='CON12')
CON13=OPTmodel.addConstr(Phvac_flex_[i] == Phvac_down_[i] + Phvac_up_[i],name='CON13')
CON14=OPTmodel.addConstr(Pbess_out_[i] == 0)
CON15=OPTmodel.addConstr(Pbess_in_[i] == 0,name='CON15')
CON16=OPTmodel.addConstr(Pbess_flex_[i] == Pbess_in_[i] + Pbess_out_[i],name='CON16')
OPTmodel.update()
ConPDC = OPTmodel.addConstr(Pdc[i] == Pflex_i[i] + float(ITload[i]), name='ConPDC')
# OPTmodel.addConstr(Tdc_i[i]==(ki*Tdc_i[i-1]+(ko*Tamb)))
#for x in Ebess_i:
#ConEBESS2 = OPTmodel.addConstr(Ebess_i[i] ==((Pbess_in[i] / 0.75) + (Pbess_out[i] * 0.75)))
cooling = np.array(pd.read_csv('Pcooling.csv'))
DRR = pd.read_csv('DR.csv')
DR = DRR.values
IT = pd.read_csv('ITpower.csv')
ITload = IT.values
newList = map(lambda x: x / 1000, ITload)
PH = pd.read_csv('Pcooling.csv')
PHVAC = PH.values
newList2 = map(lambda x: x / 1000, PHVAC)
#for y in Tdc_i:
T=pd.read_csv('TT.csv').values
OPTmodel.addConstr(Tdc_i[i]==((ki*float(T[i]))+(ko*Tamb)+(kc*float(newList[i]))-((kp*(float(newList2[i])))+(Phvac_flex[i]*3.14))))
print Tdc_i.values()
OPTmodel.addConstr(Pbess_out_[i]<=Phvac_flex[i] + Pbess_flex[i]<=Pbess_in_[i])
# Tdc_i[1:len(i)]==(Ki*Tdc_i[1:1438])+(Kc*array2[1:1438])+(Ko*Tamb))
ConBESS5 = OPTmodel.addConstr(Pbess_flex[i] == Pbess_in[i] + Pbess_out[i], name='ConBESS5')
#OPTmodel.addConstr(defIT[i]==DeferableLoad.j2 + DeferableLoad.j3)
# OPTmodel.addConstr(Pdc_base[i]==predictions[i])
ConFLEX = OPTmodel.addConstr(Pflex_i[i] == Pbess_flex[i] + Phvac_flex[i], name='ConFLEX')
PcoolingPredicted = pd.read_csv('PcoolingPredictionResult.csv')
PcoolingPredictedValue = PcoolingPredicted.values
ITPredicted = pd.read_csv('ITpredictionResult.csv')
ITPredictedValue = ITPredicted.values
ConPDCbase = OPTmodel.addConstr(Pdc_base[i] == np.transpose(ITPredictedValue[i]) + np.transpose(PcoolingPredictedValue[i]))
OPTmodel.update()
# OPTmodel.addConstr(Pdc_base[i]==prediction[i])
OPTmodel.setObjective((np.transpose(Pdc_base[i])-float(DR[i]) - (Pdc[i]) ), GRB.MINIMIZE)
OPTmodel.update()
OPTmodel.optimize()
print Pdc_base[i].X
#print Ebess_i[i].X
#print Phvac_flex[i].X
print Tdc_i[i]
print Pdc[i]
print Phvac_flex[i]
print Pbess_flex[i]
print Pbess_out[i]
print Pbess_in[i]
print Ebess_i[i]
print Pbess_flex_[i]
print Phvac_down[i]
print Phvac_up[i]
'''
def get_results(self):
"""
This function gets the results of the current optimization model
Returns
-------
"""
HVACresult = np.zeros(1,N)
BatteryResult = np.zeros(1,N)
SOC = np.zeros(1,N)
#r_Q_dot = np.zeros((self.gp.N_H, self.N_S))
#r_P = np.zeros((self.gp.N_H, self.N_S))
#r_P_self = np.zeros((self.gp.N_H, self.N_S))
#r_P_ex = np.zeros((self.gp.N_H, self.N_S))
#r_Q_dot_gas = np.zeros((self.gp.N_H, self.N_S))
#Load = np.zeros((self.gp.N_H, self.N_S))
try:
for t in range(1,N):
HVACresult[t]= Phvac_flex[t].X
BatteryResult[t]=Pbess_flex[t].X
SOC[t] = Ebess_i[t].X / Ebess_max
except:
pass
return { 'SOC' : SOC , 'BatteryResult': BatteryResult }
print OPTmodel.getVars()
# get results
Temp = {}
Battery = {}
Ebess_result = {}
ITloadd = {}
for t in range(1,N):
Temp[t] = OPTmodel.getVarByName("Tdc_i" )
Battery[t] = OPTmodel.getVarByName("PBESS_flex" )
Ebess_result[t] = OPTmodel.getVarByName("Ebess_i" )
#r_P_e[t] = model.getVarByName("P_export_%s_0" % t).X
fig, axes = plt.subplots(4, 1)
# plot elctricity
ax5 = axes[2]
ax6 = ax5.twinx()
ax5.plot( [Temp[t] for t in range(1,N)], 'g-')
ax6.plot([Ebess_result[t] for t in range(1,N)], 'b-')
ax5.set_xlabel('Time index')
ax5.set_ylabel('Power Import [W]', color='g')
ax6.set_ylabel('Power CHP [W]', color='b')
ax7 = axes[3]
ax7.plot([Battery[t] for t in range(1,N)], 'g-')
ax7.set_ylabel('Power Export [W]', color='g')
'''
print Pflex_i.values()
# print OPTmodel.getVars()
print OPTmodel.feasibility()
print OPTmodel.getObjective()
print Pdc_base.values()
'''
b = map(float, Phvac_flex)
plt.plot(b)
plt.show()
'''
#c = map(float, Pbess_flex_)
#plt.plot(c)
#plt.show()
print OPTmodel
print Tdc_i.values()
# get results
print OPTmodel.getVars()
# print OPTmodel.getAttr('EBESS_i')
status = OPTmodel.status
print status
# print Con10,Con12
print Phvac_flex.values()
print Pbess_flex.values()
print Ebess_i.values()
print OPTmodel.objval
print Tdc_i
print Pbess_in
print Pbess_out.values()
# print Pbess_flex
# print Phvac_flex
# print Ebess_i
print Pflex_i.values()
print Pbess_flex_.values()
#print OPTmodel.getVars()
print OPTmodel.feasibility()
print OPTmodel.getObjective()
print Ebess_i.values()
if OPTmodel.status == GRB.Status.INF_OR_UNBD:
# Turn presolve off to determine whether model is infeasible
# or unbounded
OPTmodel.setParam(GRB.Param.Presolve, 0)
OPTmodel.optimize()
OPTmodel.write("mymodel.lp")
if OPTmodel.status == GRB.Status.OPTIMAL:
print('Optimal objective: %g' % OPTmodel.objVal)
OPTmodel.write('model.sol')
exit(0)
elif OPTmodel.status != GRB.Status.INFEASIBLE:
print('Optimization was stopped with status %d' % OPTmodel.status)
exit(0)
# Model is infeasible - compute an Irreducible Inconsistent Subsystem (IIS)
print('')
print('Model is infeasible')
OPTmodel.computeIIS()
OPTmodel.write("model.ilp")
print("IIS written to file 'model.ilp'")
I want to plot the computed values from gurobi but when I want to get the X attribute of gurobi variable it says that AttributeError: it has no attribute 'X' and the when I cast the value from float to int it just showed me the empty plot but at the lp file I could see the result of each iteration
I am anxiously waiting for your response
cherrs
I have some code that reads an infofile and extracts information using python's regex and writes it into a new file. When I test this portion of the code individually in its own script, it works perfectly. However when I add it to the rest of my code, I get this error:
NameError: name 're' is not defined
Below is my entire code. The regex portion is obvious (all the re.search commands):
import glob
import subprocess
import os
import datetime
import matplotlib.pyplot as plt
import csv
import re
import ntpath
x = open('data.txt', 'w')
m = open('graphing_data.txt', 'w')
ckopuspath= '/Volumes/DAVIS/sfit-ckopus/ckopus'
command_init = 'sfit4Layer0.py -bv5 -fh'
subprocess.call(command_init.split(), shell=False)
with open('/Volumes/DAVIS/calpy_em27_neu/spectra_out_demo/info.txt', 'rt') as infofile: # the info.txt file created by CALPY
for count, line in enumerate(infofile):
with open('\\_spec_final.t15', 'w') as t:
lat = re.search('Latitude of location:\s*([^;]+)', line, re.IGNORECASE).group(0)
lat = lat.split()
lat = lat[3]
lat = float(lat)
lon = re.search('Longitude of location:\s*([^;]+)', line, re.IGNORECASE).group(0)
lon = lon.split()
lon = lon[3]
lon = float(lon)
date = re.search('Time of measurement \(UTC\): ([^;]+)', line).group(0)
date = date.split()
yeardate = date[4]
yeardate = yeardate.split('-')
year = int(yeardate[0])
month = int(yeardate[1])
day = int(yeardate[2])
time = date[5]
time = time.split(':')
hour = int(time[0])
minute = int(time[1])
second = float(time[2])
dur = re.search('Duration of measurement \[s\]: ([^;]+)', line).group(0)
dur = dur.split()
dur = float(dur[4])
numpoints = re.search('Number of values of one scan:\s*([^;]+)', line, re.IGNORECASE).group(0)
numpoints = numpoints.split()
numpoints = float(numpoints[6])
fov = re.search('semi FOV \[rad\] :\s*([^;]+)', line, re.IGNORECASE).group(0)
fov = fov.split()
fov = fov[3]
fov = float(fov[1:])
sza = re.search('sun Azimuth \[deg\]:\s*([^;]+)', line, re.IGNORECASE).group(0)
sza = sza.split()
sza = float(sza[3])
snr = 0.0000
roe = 6396.2
res = 0.5000
lowwav = re.search('first wavenumber:\s*([^;]+)', line, re.IGNORECASE).group(0)
lowwav = lowwav.split()
lowwav = float(lowwav[2])
highwav = re.search('last wavenumber:\s*([^;]+)', line, re.IGNORECASE).group(0)
highwav = highwav.split()
highwav = float(highwav[2])
spacebw = (highwav - lowwav)/ numpoints
d = datetime.datetime(year, month, day, hour, minute, second)
t.write('{:>12.5f}{:>12.5f}{:>12.5f}{:>12.5f}{:>8.1f}'.format(sza,roe,lat,lon,snr)) # line 1
t.write("\n")
t.write('{:>10d}{:>5d}{:>5d}{:>5d}{:>5d}{:>5d}'.format(year,month,day,hour,minute,second)) # line 2
t.write("\n")
t.write( ('{:%Y/%m/%d %H:%M:%S}'.format(d)) + "UT Solar Azimuth:" + ('{:>6.3f}'.format(sza)) + " Resolution:" + ('{:>6.4f}'.format(res)) + " Duration:" + ('{:>6.2f}'.format(dur))) # line 3
t.write("\n")
t.write('{:>21.13f}{:>26.13f}{:>24.17e}{:>12f}'.format(lowwav,highwav,spacebw,numpoints)) # line 4
t.write("\n")
calpy_path = '/Volumes/DAVIS/calpy_em27_neu/spectra_out_demo/140803/*' # the CALPY output files!
files1 = glob.glob(calpy_path)
with open(files1[count], 'r') as g:
for line in g:
wave_no, intensity = [float(item) for item in line.split()]
if lowwav <= wave_no <= highwav:
t.write(str(intensity) + '\n')
##########################
subprocess.call(['sfit4Layer0.py', '-bv5', '-fs'],shell=False) #I think this writes the summary file
# this retrieves info from summary and outputs it into data.txt (for readability)
# and graphing_data.txt (for graphing)
road = '/Volumes/DAVIS/calpy_em27_neu/spectra_out_demo/sfit4_trial' # path to summary file that is produced - not sure where this is usually*
for infile in glob.glob(os.path.join(road, 'summary*')):
lines = open(infile, 'r').readlines()
#extract info from summary
x.write('{0} {1} {2} {3} {4}'.format(fitrms, chi2, dofsall, dofstrg, iter))
x.write('\n')
x.close()
m.close()