Very new to this, but so far have had a little help in making this. It keeps giving syntax error on line 131.. and many other errors too. Im not really sure on what it is and the person who helped me create this is gone leaving me by my noob self to try and fix this.There is more to the script in the middle after -------- which decompiles the direct files but i cant post them. But i only need to figure out whats wrong with this "syntax error" so, Whats worng with this script?
#o-----------------------------------------------------------------------------o
#(-----------------------------------------------------------------------------)
#|Compatible w/ Python 3.2 |
#########################
# Configuration section #
#########################
toptablefolder = 'C:/Games/TOP/kop/scripts/table/'
toptableversion = 6
toptableencoding = 'gbk'
####end of config####
import struct
from types import *
#---------------------------------------------
class top_tsv:
name = ''
file_name = ''
list = []
version = ''
frombin_map = [('',{'v':0})]
def __init__(self, file_name=0, version=0):
if file_name == 0:
self.file_name = toptablefolder+self.__class__.__name__
else:
self.file_name = file_name
if (version == 0):
self.version = toptableversion
else:
self.version = version
self.list = []
self.frombin_map = [('',{'v':0})]
def unencrypt(self):
item = []
#Load data
file = open(self.file_name+'.bin', 'rb')
data = bytearray(file.read())
file.close()
i = 0
array_size = int(list(struct.unpack_from('<i',data[i:i+4]))[0])
rdata = data[i:i+4]
data = data[i+4:len(data)]
m = 0
k = 0
l = len(data)//array_size
#2.2 encryption
if ((self.version >= 5) and ((self.__class__.__name__ != 'magicsingleinfo') and (self.__class__.__name__ != 'magicgroupinfo') and (self.__class__.__name__ != 'resourceinfo') and (self.__class__.__name__ != 'terraininfo'))):
newbytes = bytes.fromhex('98 9D 9F 68 E0 66 AB 70 E9 D1 E0 E0 CB DD D1 CB D5 CF')
while(k<l):
item = data[i:i+array_size]
i = i+array_size
if ((self.version >= 5) and ((self.__class__.__name__ != 'magicsingleinfo') and (self.__class__.__name__ != 'magicgroupinfo') and (self.__class__.__name__ != 'resourceinfo') and (self.__class__.__name__ != 'terraininfo'))):
for m,c in enumerate(item):
j = m % len(newbytes)
item[m] = ((item[m] - newbytes[j]) + 256) % 256
rdata = rdata + item
k = k + 1
m = 0
file = open(self.file_name+'-un.bin', 'wb')
file.write(rdata)
file.close()
def load_bin_data(self):
array = []
item = []
addresses = []
#Load structure (calculate size)
struct_type_map={
"char":"c",
"byte":"b",
"ubyte":"B",
"_bool":"?",
"short":"h",
"ushort":"H",
"int":"i",
"uint":"I",
"long":"l",
"ulong":"L",
"quad":"q",
"uquad":"Q",
"float":"f",
"double":"d",
"str":"s",
"color":"B",
"rcolor":"B",
}
struct_vars = {'t':'','s':1,'l':1,'stp':1,'f':-1,'v':-1,'lpad':0,'rpad':0,'sa':-1,'sap':-1,'ea':-1,'eap':-1,'st':'','lm':0,'sm':0,'smb':0,'smea':0,'sv':0,'func':0}
#initialize addresses
struct_init_address = 0
struct_limit_address = 0
struct_marked_addresses = [0,0,0,0,0,0]
struct_func_indexes = []
for i, v in enumerate(list(zip(*self.frombin_map))[1]):
struct_item = struct_vars.copy()
struct_item.update(v)
if struct_item['smb']>=1:
struct_marked_addresses[struct_item['smb']] = struct_init_address
if struct_item['lm']>=1:
struct_init_address = struct_marked_addresses[struct_item['lm']]
if struct_item['sm']>=1:
struct_marked_addresses[struct_item['sm']] = struct_init_address
if (type(struct_item['func']) == FunctionType):
struct_func_indexes.append(i)
elif (struct_item['v'] == -1):
if type(struct_item['t']) == tuple:
struct_item['s'] = len(struct_item['t'])
struct_item['st'] = []
for j,t in enumerate(struct_item['t']):
struct_item['st'].append(struct_type_map[struct_item['t'][j]])
struct_item['st'] = tuple(struct_item['st'])
struct_item['s'] = len(struct_item['st'])
else:
struct_item['st'] = struct_type_map[struct_item['t']]
if ((struct_item['t'] == 'color') or (struct_item['t'] == 'rcolor')):
struct_item['s'] = 3
struct_item['sa'] = struct_init_address
struct_item['sap'] = struct_item['sa'] + struct_item['lpad']
struct_item['ea'] = struct_item['sap']
if type(struct_item['t']) == tuple:
for j,t in enumerate(struct_item['t']):
struct_item['ea'] = struct_item['ea'] + struct.calcsize(struct_item['st'][j]) * ((struct_item['stp'] * struct_item['l'])-(struct_item['stp']-1))
else:
struct_item['ea'] = struct_item['ea'] + struct.calcsize(struct_item['st']) * ((struct_item['s'] * struct_item['stp'] * struct_item['l'])-(struct_item['stp']-1))
if struct_item['smea']>=1:
struct_marked_addresses[struct_item['smea']] = struct_item['ea']
struct_item['eap'] = struct_item['ea'] + struct_item['rpad']
struct_init_address = struct_item['eap']
if (struct_init_address > struct_limit_address):
struct_limit_address = struct_init_address
#print(struct_item)
self.frombin_map[i] = (self.frombin_map[i][0],struct_item)
struct_size = struct_limit_address
#Load data
file = open(self.file_name+'.bin', 'rb')
data = bytearray(file.read())
file.close()
i = 0
k = 0
array_size = int(list(struct.unpack_from('<i',data[i:i+4]))[0])
if array_size != struct_size:
print(self.file_name+'.bin: Actual array size ('+str(array_size)+') doesn''t match structure size ('+str(struct_size)+')')
raise 'Size error.'
data = data[i+4:len(data)]
m = 0
k = 0
l = len(data)//struct_size
#2.2 encryption
if ((self.version >= 5) and ((self.__class__.__name__ != 'magicsingleinfo') and (self.__class__.__name__ != 'magicgroupinfo') and (self.__class__.__name__ != 'resourceinfo') and (self.__class__.__name__ != 'terraininfo'))):
newbytes = bytes.fromhex('98 9D 9F 68 E0 66 AB 70 E9 D1 E0 E0 CB DD D1 CB D5 CF')
while(k<l):
item = data[i:i+struct_size]
i = i+struct_size
if ((self.version >= 5) and ((self.__class__.__name__ != 'magicsingleinfo') and (self.__class__.__name__ != 'magicgroupinfo') and (self.__class__.__name__ != 'resourceinfo') and (self.__class__.__name__ != 'terraininfo'))):
for m,c in enumerate(item):
j = m % len(newbytes)
item[m] = ((item[m] - newbytes[j]) + 256) % 256
array.append(item)
k = k + 1
m = 0
#Associate the data with the structure
self.list = []
self.list.append(list(zip(*self.frombin_map))[0])
for y,rawrow in enumerate(array):
row = []
for x,cell_struct in enumerate(list(zip(*self.frombin_map))[1]):
if type(cell_struct['func']) == FunctionType:
cell = []
cell.append('0')
row.append(self.transformtostr(cell,**cell_struct))
else:
cell = []
if (cell_struct['v'] == -1):
i = cell_struct['sap']
for j in range(cell_struct['l']):
processed_data=list(struct.unpack_from('<'+str((cell_struct['s']//len(cell_struct['st']))*cell_struct['stp'])+"".join(cell_struct['st']),rawrow,i))[::cell_struct['stp']]
#if (x == 4) and (y == 70):
# #print(cell_struct['s'])
cell.append(processed_data)
i=i+cell_struct['s']*struct.calcsize("".join(cell_struct['st']))*cell_struct['stp'] #sizeof here
if (cell_struct['t'] == 'rcolor'):
cell[0].reverse()
else:
cell.append(cell_struct['v'])
#if y == 70:
# print(cell) #'s':0x02,'l':0x08
row.append(self.transformtostr(cell,**cell_struct))
self.list.append(row)
for x,row in enumerate(self.list):
if x>0:
for func_index in struct_func_indexes:
self.list[x][func_index] = list(zip(*self.frombin_map))[1][func_index]['func'](func_index, row)
deletions = 0
for z,struct_item_name in enumerate(list(zip(*self.frombin_map))[0]):
if (struct_item_name == ''):
del self.list[x][z-deletions]
deletions = deletions + 1
return
def i(self,x):
for i,v in enumerate(self.list):
if(x==v):
return i
break
return -1
def j(self,x):
for i,v in enumerate(list(zip(*self.frombin_map))[0]):
if(x==v):
return i
break
return -1
def remap(self,v):
for i,n in enumerate(list(zip(*v))[0]):
if self.j(n) == -1:
self.frombin_map.append((list(zip(*v))[0][i],list(zip(*v))[1][i]))
else:
if (list(zip(*v))[1][i] == {}):
del self.frombin_map[self.j(n)]
else:
self.frombin_map[self.j(n)] = (list(zip(*v))[0][i],list(zip(*v))[1][i])
def bintotsv(self):
file = open(self.file_name+'.txt', 'wt', encoding=toptableencoding)
for i,a in enumerate(self.list):
a=list(a)
if (i==0):
deletions = 0
for z,item in enumerate(a[:]):
if (item == ''):
del a[z-deletions]
deletions = deletions + 1
file.write('//'+'\t'.join(a))
else:
#print(a)
file.write('\n'+'\t'.join(a))
file.close()
return self
def trim_nullbytestr(string, flag=0): #flag=1, return "0" when string is empty
result = string
for i,byte in enumerate(string[:]):
if byte == '\00':
result = string[:i]
break
if (flag & 1) and ((string == '\00') or (string == '')):
result = '0'
return result
def transformtostr(self,result,t,s,l,stp,f,v,ea,eap,lpad,rpad,sa,sap,st,sm,lm,smb,smea,sv,func):
if v != -1:
return str(v)
_type = t
j = 0
for n,v in enumerate(result[:]):
m = 0
if type(t) == tuple:
_type = t[j]
is_integer = not((_type=="float") or (_type=="double") or (_type=="str") or (_type=="char") or (_type=="_bool"))
if is_integer:
if f==-1:
f=0
for k,w in enumerate(v[:]):
if is_integer:
#Result: flag=0x0001 = remove FF's, #flag=0x0002 = cut file defect bytes, #flag=0x0004 = remove 0's
if ((((f & 0x4) and (w == 0))) or
((f & 0x1) and ((w == (2**8)-1) or (w == (2**16)-1) or (w == (2**32)-1) or (w == (2**64)-1) or (w == -1))) or
((f & 0x2) and ((((_type=="byte") or (_type=="ubyte")) and (w == (2**8)-51)) or (((_type=="short") or (_type=="ushort")) and (w == (2**16)-12851)) or (((_type=="long") or (_type=="ulong")) and (w == (2**32)-842150451)) or (((_type=="quad") or (_type=="uquad")) and (w == (2**64)-3617008641903833651))))):
del result[j][m]
m=m-1
else:
if (f & 0x2 and w == 0):
result[j] = result[j][:m]
break
result[j][m]=str(result[j][m])
m=m+1
if (_type=="float"):
if f==-1:
f=3
result[j][k]=str(round(float(w),f))
if (_type=="str"):
if f==-1:
f=0
if (w[0] == 205) and (w[1] == 205):
result[j][m] = ''
else:
result[j][m] = w.decode('gbk','ignore')
for o,x in enumerate(result[j][m]):
if x == '\00':
result[j][m] = result[j][m][:o]
break
#result[j][m] = result[j][m].strip()
if ((f & 1) and (result[j][m] == '')):
result = '0'
if (result[j] != '0'):
result[j] = ','.join(result[j])
if is_integer:
if (result[j] == ''):
result[j] = '0'
j=j+1
if (_type=="str"):
result = ','.join(result)
if ((f & 1) and (result == '')):
result = '0'
else:
result = ';'.join(result)
if is_integer:
if (result == ''):
result = '0'
return result
]
for c in toptablebins:
myc = c()
try:
myc.load_bin_data()
except IOError:
print('"'+myc.file_name+'.bin'+'" doesn''t exist, skipping...')
myc.bintotsv()
these are a few lines below and above line 131 which is the error
for j,t in enumerate(struct_item['t']):
struct_item['st'].append(struct_type_map[struct_item['t'][j]])
struct_item['st'] = tuple(struct_item['st'])
struct_item['s'] = len(struct_item['st'])
else: <--- this is 131
struct_item['st'] = struct_type_map[struct_item['t']]
if ((struct_item['t'] == 'color') or (struct_item['t'] == 'rcolor')):
struct_item['s'] = 3
struct_item['sa'] = struct_init_address
struct_item['sap'] = struct_item['sa'] + struct_item['lpad']
struct_item['ea'] = struct_item['sap']
if type(struct_item['t']) == tuple:
for j,t in enumerate(struct_item['t']):
struct_item['ea'] = struct_item['ea'] + struct.calcsize(struct_item['st'][j]) * ((struct_item['stp'] * struct_item['l'])-(struct_item['stp']-1))
It doesn't look like the you have the proper indention. Remember, in Python, whitespace is very very important.
The problem section of code should probably start looking something like:
elif (struct_item['v'] == -1):
if type(struct_item['t']) == tuple:
struct_item['s'] = len(struct_item['t'])
struct_item['st'] = []
for j,t in enumerate(struct_item['t']):
struct_item['st'].append(struct_type_map[struct_item['t'][j]])
struct_item['st'] = tuple(struct_item['st'])
struct_item['s'] = len(struct_item['st'])
This seems a bit weird:
l = len(data)//struct_size
You really need to post the traceback the the interpreter is giving you - it makes finding the error much easier.
Related
I have a certain number of matrix in a list and when I try to append a matrix to the NumPy array It only appends the specified row and when I try to edit the code to append the whole matrix It keep returning the following error:
Traceback (most recent call last):
File "so.py", line 129, in <module>
parents = selection(cal_fitness(t1, t2, t3, matlist, 300, arr), 2, matlist) #
File "so.py", line 124, in selection
parents[i, :] = population[max_fitness_idx[0][0]]
ValueError: could not broadcast input array from shape (3,5) into shape (5,)
The error related function:
def selection(fitness, num_parents, population):
fitness = list(fitness)
parents = numpy.empty((num_parents, len(population)))
for i in range(num_parents):
max_fitness_idx = numpy.where(fitness == numpy.max(fitness))
#parents[i, :] = population[max_fitness_idx[0][0]][2]
parents[i, :] = population[max_fitness_idx[0][0]]
fitness[max_fitness_idx[0][0]] = -999999
return parents.astype(int)
parents = selection(cal_fitness(t1, t2, t3, matlist, 300, arr), 2, matlist)
print(parents)
The full code:
import numpy
import random
import pandas
wsn = numpy.arange(1, 6)
taskn = 3
t1 = numpy.random.randint(30, 200, size=len(wsn))
t2 = numpy.random.randint(30, 200, size=len(wsn))
t3 = numpy.random.randint(30, 200, size=len(wsn))
# print('\nGenerated Data:\t\n\nNumber \t Task 1 \t Task 2 \t Task 3\n')
ni = min(len(t1), len(t2), len(t3))
# for i in range(ni):
# print('\t {0} \t {1} \t {2} \t\t {3}\n'.format(wsn[i], t1[i], t2[i], t3[i]))
# print('\n\n')
qmin = 50
qmax = 140
for i in range(len(t1)):
if t1[i] <= qmin or t1[i] >= qmax:
# t1=numpy.delete(t1,i)
t1[i] = 0
for i in range(len(t2)):
if t2[i] <= qmin or t2[i] >= qmax:
# t2=numpy.delete(t2,i)
t2[i] = 0
for i in range(len(t3)):
if t3[i] <= qmin or t3[i] >= qmax:
# t3=numpy.delete(t3,i)
t3[i] = 0
i = 0
m = max(len(t1), len(t2), len(t3))
if t1[i] == 0 and t2[i] == 0 and t3[i] == 0:
t1 = numpy.delete(t1, i)
t2 = numpy.delete(t2, i)
t3 = numpy.delete(t3, i)
i += 1
solperpop = len(wsn)
gen = 20
j = 0
pop_size = (taskn, solperpop)
# print('population size: {}'.format(pop_size))
# for j in range(ni):
# pop_size=list(solperpop,taskn)
matlist = list()
# print('\n\n')
i = 0
k = 0
nbrofindv = 5
arr = []
for i in range(nbrofindv):
init_pop = numpy.zeros(pop_size, dtype=int)
init_pop = init_pop.astype(int)
k = 0
l = 0
for k in range(taskn):
l = random.randrange(solperpop - 1)
init_pop[k][l] = 1
arr.append(l)
matlist.append(init_pop)
pandas.set_option('display.max_columns', None)
pandas.set_option('display.width', None)
zipped = pandas.DataFrame(list(zip(*matlist)))
# , columns=['Individual 1', 'Individual 2', 'Individual 3', 'Individual 4', 'Individual 5'])
print(zipped)
print('\n\n')
i = 0
for i in range(len(wsn)):
if t1[i] == 0:
if init_pop[0][i] != 0:
init_pop[0][i] == 0
if t2[i] == 0:
if init_pop[1][i] != 0:
init_pop[1][i] == 0
if t3[i] == 0:
if init_pop[2][i] != 0:
init_pop[2][i] == 0
def cal_fitness(task1, task2, task3, matix, mmax, array):
fitness = numpy.empty(len(matix))
S1 = numpy.empty(len(matix), dtype=int)
z = 0
for i in range(len(matix)):
S1[i] = task1[array[0 + z]] + task2[array[1 + z]] + task3[array[2 + z]]
z += 3
if S1[i] <= mmax:
fitness[i] = S1[i]
else:
fitness[i] = 0
return fitness.astype(int)
fitness = cal_fitness(t1, t2, t3, matlist, 300, arr)
def selection(fitness, num_parents, population):
fitness = list(fitness)
parents = numpy.empty((num_parents, len(population)))
for i in range(num_parents):
max_fitness_idx = numpy.where(fitness == numpy.max(fitness))
#parents[i, :] = population[max_fitness_idx[0][0]][2]
parents[i, :] = population[max_fitness_idx[0][0]]
fitness[max_fitness_idx[0][0]] = -999999
return parents.astype(int)
parents = selection(cal_fitness(t1, t2, t3, matlist, 300, arr), 2, matlist)
print(parents)
print('\n\n')
def crossover(parents, num_offsprings):
offsprings = numpy.empty((num_offsprings, parents.shape[1]))
crossover_point = int(parents.shape[1] / 2)
crossover_rate = 0.5
i = 0
while parents.shape[0] < num_offsprings:
parent1_index = i % parents.shape[0]
parent2_index = (i + 1) % parents.shape[0]
x = random.random()
if x > crossover_rate:
continue
parent1_index = i % parents.shape[0]
parent2_index = (i + 1) % parents.shape[0]
offsprings[i, 0:crossover_point] = parents[parent1_index, 0:crossover_point]
offsprings[i, crossover_point:] = parents[parent2_index, crossover_point:]
i += 1 # <== modified
return offsprings # <== modified
print(crossover(parents, 2)) # <== modified
This issue is related to dimensions that specified for parents, which is 2d now so must be modified to 3d. So, IIUC, you can achieve this aim by:
def selection(fitness, num_parents, population):
fitness = list(fitness)
# parents shape is modified (a new axis with the needed length is added)
parents = numpy.empty((num_parents, len(population[0]), len(population[0][0]))) # <==
for i in range(num_parents):
max_fitness_idx = numpy.where(fitness == numpy.max(fitness))
parents[i, :] = population[max_fitness_idx[0][0]]
fitness[max_fitness_idx[0][0]] = -999999
return parents.astype(int)
I am trying to create an indicator that will find all the divergences between 2 signals.
The output of the function so far looks like this
But the problem is that is painfully slow when I am trying to use it with long signals. Could any of you guys help me to make it faster if is possible?
My code:
def find_divergence(price: pd.Series, indicator: pd.Series, width_divergence: int, order: int):
div = pd.DataFrame(index=range(price.size), columns=[
f"Bullish_{width_divergence}_{order}",
f"Berish_{width_divergence}_{order}"
])
div[f'Bullish_idx_{width_divergence}_{order}'] = False
div[f'Berish_idx_{width_divergence}_{order}'] = False
def calc_argrelextrema(price_: np.numarray):
return argrelextrema(price_, np.less_equal, order=order)[0]
price_ranges = []
for i in range(len(price)):
price_ranges.append(price.values[0:i + 1])
f = []
with ThreadPoolExecutor(max_workers=16) as exe:
for i in price_ranges:
f.append(exe.submit(calc_argrelextrema, i))
prices_lows = SortedSet()
for r in concurrent.futures.as_completed(f):
data = r.result()
for d in reversed(data):
if d not in prices_lows:
prices_lows.add(d)
else:
break
price_lows_idx = pd.Series(prices_lows)
for idx_1 in range(price_lows_idx.size):
min_price = price[price_lows_idx[idx_1]]
min_indicator = indicator[price_lows_idx[idx_1]]
for idx_2 in range(idx_1 + 1, idx_1 + width_divergence):
if idx_2 >= price_lows_idx.size:
break
if price[price_lows_idx[idx_2]] < min_price:
min_price = price[price_lows_idx[idx_2]]
if indicator[price_lows_idx[idx_2]] < min_indicator:
min_indicator = indicator[price_lows_idx[idx_2]]
consistency_price_rd = min_price == price[price_lows_idx[idx_2]]
consistency_indicator_rd = min_indicator == indicator[price_lows_idx[idx_1]]
consistency_price_hd = min_price == price[price_lows_idx[idx_1]]
consistency_indicator_hd = min_indicator == indicator[price_lows_idx[idx_2]]
diff_price = price[price_lows_idx[idx_1]] - price[price_lows_idx[idx_2]] # should be neg
diff_indicator = indicator[price_lows_idx[idx_1]] - indicator[price_lows_idx[idx_2]] # should be pos
is_regular_divergence = diff_price > 0 and diff_indicator < 0
is_hidden_divergence = diff_price < 0 and diff_indicator > 0
if is_regular_divergence and consistency_price_rd and consistency_indicator_rd:
div.at[price_lows_idx[idx_2], f'Bullish_{width_divergence}_{order}'] = (price_lows_idx[idx_1], price_lows_idx[idx_2])
div.at[price_lows_idx[idx_2], f'Bullish_idx_{width_divergence}_{order}'] = True
elif is_hidden_divergence and consistency_price_hd and consistency_indicator_hd:
div.at[price_lows_idx[idx_2], f'Berish_{width_divergence}_{order}'] = (price_lows_idx[idx_1], price_lows_idx[idx_2])
div.at[price_lows_idx[idx_2], f'Berish_idx_{width_divergence}_{order}'] = True
return div
I have this code, to convert .VCF files to .GENO files. To test it, I just used the smallest file on my laptop, but for all, I need a bigger machine, and it would be really nice, if it wouldn't take weeks to run. It works perfectly on my laptop (I7 4th gen), but really slow on our workstation server (have 48 cores, but slower). How can I modify the code, to use more cores? Thank you in advance.
The code:
import allel
import pandas as pd
import numpy as np
from time import process_time
import numba as nb
#nb.jit(forceobj=True)
def create_chrpos(data, n):
chr_pos = []
chr_pos = np.array(chr_pos, dtype=np.int32)
for i in range(len(data)):
if data['chr'][i] == n:
if i == 0:
chr_pos = data['pos'][0]
else:
a = data['pos'][i]
chr_pos = np.append(chr_pos, [a])
return chr_pos
#nb.njit
def create_needed_pos(chr_pos, pos):
needed_pos = nb.typed.List.empty_list(nb.int32)
for i in range(len(chr_pos)):
for k in range(len(pos)):
if chr_pos[i] == pos[k]:
if i == k == 1:
needed_pos = nb.typed.List([pos[k]])
else:
needed_pos.append(pos[k])
return needed_pos
#nb.njit
def create_needed_index(chr_pos, pos):
needed_index = nb.typed.List.empty_list(nb.int32)
for i in range(len(chr_pos)):
for k in range(len(pos)):
if chr_pos[i] == pos[k]:
if i == k == 1:
needed_index = nb.typed.List([pos[k]])
else:
needed_index.append(pos[k])
return needed_index
#nb.njit
def create_mat(geno):
# create matrix as np.uint8 (1 byte) instead of list of python integers (8 byte)
# also no need to dynamically resize / increase list size
geno_mat = np.zeros((len(geno[:, 0]), len(geno[1, :])), dtype=np.uint8)
for i in np.arange(len(geno[:, 0])):
for k in np.arange(len(geno[1, :])):
g = geno[i, k]
# nested ifs to avoid duplicate comparisons
if g[0] == 0:
if g[1] == 0:
geno_mat[i, k] = 2
elif g[1] == 1:
geno_mat[i, k] = 1
else:
geno_mat[i, k] = 9
elif g[0] == 1:
if g[1] == 0:
geno_mat[i, k] = 1
elif g[1] == 1:
geno_mat[i, k] = 0
else:
geno_mat[i, k] = 9
else:
geno_mat[i, k] = 9
return geno_mat
def genotyping(geno, pos, chr_pos):
needed_pos = create_needed_pos(chr_pos, pos)
create_needed_index(chr_pos, pos)
mat = create_mat(geno)
list_difference = [item for item in chr_pos if item not in needed_pos]
needed_pos_list = list(needed_pos)
matrix_df = pd.DataFrame(mat, dtype=int, index=pos)
filtered_geno_dataframe = matrix_df.loc[needed_pos_list, :]
missing_positions_df = pd.DataFrame(index=list_difference, columns=np.arange(2054))
missing_positions_df.fillna(2, inplace=True)
finaldataframe = pd.concat([filtered_geno_dataframe, missing_positions_df])
finaldataframe.sort_index(axis=0, inplace=True)
final_mat = finaldataframe.to_numpy(dtype=np.int32)
return final_mat
def write_first_chr(genotype):
with open('test_1.geno', 'wb') as fout: # Note 'wb' instead of 'w'
np.savetxt(fout, genotype, delimiter="", fmt='%d')
fout.seek(-2, 2)
fout.truncate()
def write_remaining_chr(genotype):
with open('test_1.geno', 'a') as fout: # Note 'wb' instead of 'w'
np.savetxt(fout, genotype, delimiter="", fmt='%d')
fout.seek(-2, 2)
fout.truncate()
if __name__ == "__main__":
t1_start = process_time()
data = pd.read_csv('REICH_1KG.snp', delimiter=r"\s+")
data.columns = ['ID', "chr", "pyspos", "pos", "Ref", "Alt"]
samples = open("sample_list_test.txt")
for i, line in enumerate(samples):
strip_line = line.strip()
n = i + 1
chr_pos = create_chrpos(data, n)
geno = allel.read_vcf(strip_line, fields=("calldata/GT",))["calldata/GT"]
pos = allel.read_vcf(strip_line, fields=("variants/POS",))["variants/POS"]
genotype = genotyping(geno, pos, chr_pos)
if i + 1 == 1:
print("First chromosome done")
write_first_chr(genotype)
else:
write_remaining_chr(genotype)
print("Done:Chr number:", n)
print("Finished genotyping")
t1_stop = process_time()
print("Ennyi idő kellett teszt1:", t1_stop - t1_start)
import os
import sys
import math
import cvxopt as cvx
import picos as pic
import pandas as pd
import matplotlib.pyplot as plt
from gurobipy import *
from statsmodels.tsa.arima_model import ARIMA
import numpy as np
from scipy import *
#import DeferableLoad
OPTmodel = Model('OPTIMIZER')
#general parameters
Tamb =22
N = 1440 # maximum iteration
i = range(1, N)
COP= 3.4 # Coeffient of performance
'''
Prediction need to be added here
'''
# Datacenter room defintion
R = 10 #length of room
B = 7
H = 9 #Height of room
L = 10
dT = 60
A = 2*((L*B)+(B*H)+(H*L))
Thick = 0.33 # thickness of wall
k = 0.7 # thermal conductivity of wall
mAir = 1.2 * (L * B * H)
C = 718
landa = k * A / Thick
a0 = 0.05 / dT
a1 = 1
ki = math.exp(-(landa * 60) / (mAir * C)) # value that constant and its related to property of room
kc = (1 - ki) * a0
ko = (1 - ki) * a1
kp = (1 - ki) * (COP / landa)
Tmin= 18
Tmax= 27
Tamb= 22
PcoolingRated = 100
Pbess_rated = 30.462
Pbess_ratedN = -30.462
Ebess_min = 0
Ebess_max = 300
with open ('Pcooling.csv','r') as f:
Pcooling = []
for line in f:
Pcooling.append(line)
f.close()
with open ('ITpower.csv','r') as f1:
ITload = []
for line1 in f1:
ITload.append(line1)
f1.close()
with open ('DR.csv','r') as f2:
DR =[]
for line2 in f2:
DR.append(line2)
f2.close()
print ITload
print Pcooling
print DR
for i in range(1,200):
for it in range(1, 1440):
Tm = np.empty(1440)
Tm.fill(18)
TmA = np.empty(1440)
TmA.fill(27)
Phvac_flex = {}
Phvac_up = {}
Phvac_down_= {}
Phvac_up_ = {}
Pbess_out_ = {}
Pbess_in_ = {}
Phvac_down = {}
Pbess_flex_ = {}
Pbess_flex = {}
Phvac_flex_ = {}
Pbess_in = {}
Pdc = {}
Pdc_base = {}
Pflex_i = {}
Tdc_i = {}
Pbess_out ={}
Ebess_i = {}
Phvac_flex[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS,name="PHVAC_flex"+str(i))
Phvac_up[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PHVAC_up" + str(i))
Phvac_up_[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PHVAC_up_" + str(i))
Phvac_down_[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PHVAC_down_" + str(i))
Pbess_out_[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PBESS_out_" + str(i))
Pbess_in_[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PBESS_in_" + str(i))
Phvac_down[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PHVAC_down" + str(i))
Pbess_flex_[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PBESS_flex_" + str(i))
Pbess_flex[i] = OPTmodel.addVar(lb=-GRB.INFINITY,ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PBESS_flex" + str(i))
Phvac_flex_[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PHVAC_flex_" + str(i))
Pbess_in[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PBESS_in" + str(i))
Pdc[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PDC" + str(i))
Pdc_base[i] = OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PDC_base" + str(i))
Pflex_i[i]= OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="Pflex_i" + str(i))
Tdc_i[i]= OPTmodel.addVar(ub=GRB.INFINITY,vtype = GRB.CONTINUOUS, name = "Tdc_i" + str(i))
Pbess_out[i] = OPTmodel.addVar(lb=-GRB.INFINITY,ub=GRB.INFINITY,vtype=GRB.CONTINUOUS, name="PBESS_out" + str(i))
Ebess_i[i]= OPTmodel.addVar(ub=GRB.INFINITY,vtype=GRB.CONTINUOUS,name="Ebess_i" + str(i))
Pflex_i[1] = 0
Pflex_i[1] = 0
Tdc_i[0] = 18
Phvac_flex[1] = 0
# Phvac_flex_[1] = 0
Phvac_down[1] = 0
Phvac_up[1] = 0
Phvac_down_[1] = 0
Phvac_up_[1] = 0
# Phvac_down_pos[1] = 0
# Phvac_up_pos(1) = 0;
Pbess_flex[1] = 0
# Pbess_flex_[1] = 0
Pbess_out[1] = 0
Pbess_in[1] = 0
# Pbess_out_[1] = 0
Pbess_in_[1] = 0
# Pbess_out_pos[1] = -250
# Pbess_in_pos(1) = 250;
Ebess_i[1] = 150
OPTmodel.update()
'''
if float(DR[i]) > 0:
Phvac_down_[i] = 0
Phvac_up_[i] = float(DR[i])
Pbess_out_[i] = 0
Pbess_in_[i] = float(DR[i])
#Pbess_flex_[i] = Pbess_in_[i] + Pbess_out_[i]
#Phvac_flex_[i] = Phvac_down_[i] + Phvac_up_[i]
OPTmodel.update()
elif float(DR[i]) < 0:
Phvac_down_[i] = float(DR[i])
Phvac_up_[i] = 0
#Phvac_flex_[i] = Phvac_down_[i] + Phvac_up_[i]
Pbess_out_[i] = float(DR[i])
Pbess_in_[i] = 0
#Pbess_flex_[i] = Pbess_in_[i] + Pbess_out_[i]
OPTmodel.update()
else:
Phvac_down_[i] = 0
Phvac_up_[i] = 0
Phvac_flex_[i] = Phvac_down_[i] + Phvac_up_[i]
Pbess_out_[i] = 0
Pbess_in_[i] = 0
Pbess_flex_[i] = Pbess_in_[i] + Pbess_out_[i]
OPTmodel.update()
'''
#print Phvac_up.values()
#print Phvac_flex_[i]
print OPTmodel
OPTmodel.update()
ConHVAC1 = OPTmodel.addConstr(Phvac_flex[i] == Phvac_up[i] + Phvac_down[i], name='ConHVAC1')
ConHVAC2 = OPTmodel.addConstr(0 <= Phvac_flex[i] , name='ConHVAC2')
ConHVAC3 = OPTmodel.addConstr(Phvac_flex[i] <= PcoolingRated, name='ConHVAC3')
PH = pd.read_csv('Pcooling.csv')
PHVAC = PH.values
newList2 = map(lambda x: x / 1000, PHVAC)
p=[]
p=PcoolingRated-newList2[i]
#CONHVAC4 = OPTmodel.addConstr(Phvac_up[i]==np.minimum((Phvac_up_[i]),(float(newList2[i]))))
#Phvac_u(1:MaxIter) == min(Phvac_u_(1:MaxIter), (repelem(Phvac_max, MaxIter) - (Pcooling(1:MaxIter)'/1000)))
ConTemp1 = OPTmodel.addConstr(Tm[it] <= Tdc_i[i] <= TmA[it], name='ConTemp1')
ConBESS1 = OPTmodel.addConstr(Pbess_ratedN <= Pbess_flex[i] <= Pbess_rated, name='ConBESS1')
ConBESS2 = OPTmodel.addConstr(Pbess_flex[i] == Pbess_in[i] + Pbess_out[i], name='ConBESS2')
ConBESS3 = OPTmodel.addConstr(0 <= Pbess_in[i] <= min(Pbess_rated, Pbess_in_[i]), name='ConBESS3')
ConBESS4 = OPTmodel.addConstr(np.maximum(Pbess_ratedN,Pbess_out_[i]) <= Pbess_out[i]<=0 , name='ConBESS4') # need to modifty
ConEBESS1 = OPTmodel.addConstr(Ebess_min <= Ebess_i[i], name='ConEBESS1')
ConEBESS2 = OPTmodel.addConstr(Ebess_i[i] <= Ebess_max, name='ConEBESS2')
D = pd.read_csv('DR.csv').values
DRN = map(lambda x: x / 1000, D)
PDRN=map(lambda x: x / 4.8, DRN)
if float((PDRN[i])) > 0:
CON1 = OPTmodel.addConstr(Pbess_flex_[i] == Pbess_in_[i] + Pbess_out_[i],'CON1')
CON2 = OPTmodel.addConstr(Phvac_flex_[i] == Phvac_up_[i] + Phvac_down_[i],'CON2')
CON3=OPTmodel.addConstr(Phvac_down_[i] == 0, name='CON3')
CON4=OPTmodel.addConstr(Phvac_up_[i] == float((PDRN[i])),name='CON4')
CON5=OPTmodel.addConstr(Pbess_out_[i] == 0,name='CON5')
CON6=OPTmodel.addConstr(Pbess_in_[i] == float((PDRN[i])),name='CON6')
elif float(np.transpose(PDRN[i])) < 0:
CON7=OPTmodel.addConstr(Phvac_down_[i] == float(np.transpose(PDRN[i])),name='CON7')
CON8=OPTmodel.addConstr(Phvac_up_[i] == 0,name='CON8')
# Phvac_flex_[i] = Phvac_down_[i] + Phvac_up_[i]
CON9=OPTmodel.addConstr(Pbess_out_[i] == float((PDRN[i])),name='CON9')
CON10=OPTmodel.addConstr(Pbess_in_[i] == 0,name='CON10')
else:
CON11=OPTmodel.addConstr(Phvac_down_[i] == 0,name='CON11')
CON12=OPTmodel.addConstr(Phvac_up_[i] == 0,name='CON12')
CON13=OPTmodel.addConstr(Phvac_flex_[i] == Phvac_down_[i] + Phvac_up_[i],name='CON13')
CON14=OPTmodel.addConstr(Pbess_out_[i] == 0)
CON15=OPTmodel.addConstr(Pbess_in_[i] == 0,name='CON15')
CON16=OPTmodel.addConstr(Pbess_flex_[i] == Pbess_in_[i] + Pbess_out_[i],name='CON16')
OPTmodel.update()
ConPDC = OPTmodel.addConstr(Pdc[i] == Pflex_i[i] + float(ITload[i]), name='ConPDC')
# OPTmodel.addConstr(Tdc_i[i]==(ki*Tdc_i[i-1]+(ko*Tamb)))
#for x in Ebess_i:
#ConEBESS2 = OPTmodel.addConstr(Ebess_i[i] ==((Pbess_in[i] / 0.75) + (Pbess_out[i] * 0.75)))
cooling = np.array(pd.read_csv('Pcooling.csv'))
DRR = pd.read_csv('DR.csv')
DR = DRR.values
IT = pd.read_csv('ITpower.csv')
ITload = IT.values
newList = map(lambda x: x / 1000, ITload)
PH = pd.read_csv('Pcooling.csv')
PHVAC = PH.values
newList2 = map(lambda x: x / 1000, PHVAC)
#for y in Tdc_i:
T=pd.read_csv('TT.csv').values
OPTmodel.addConstr(Tdc_i[i]==((ki*float(T[i]))+(ko*Tamb)+(kc*float(newList[i]))-((kp*(float(newList2[i])))+(Phvac_flex[i]*3.14))))
print Tdc_i.values()
OPTmodel.addConstr(Pbess_out_[i]<=Phvac_flex[i] + Pbess_flex[i]<=Pbess_in_[i])
# Tdc_i[1:len(i)]==(Ki*Tdc_i[1:1438])+(Kc*array2[1:1438])+(Ko*Tamb))
ConBESS5 = OPTmodel.addConstr(Pbess_flex[i] == Pbess_in[i] + Pbess_out[i], name='ConBESS5')
#OPTmodel.addConstr(defIT[i]==DeferableLoad.j2 + DeferableLoad.j3)
# OPTmodel.addConstr(Pdc_base[i]==predictions[i])
ConFLEX = OPTmodel.addConstr(Pflex_i[i] == Pbess_flex[i] + Phvac_flex[i], name='ConFLEX')
PcoolingPredicted = pd.read_csv('PcoolingPredictionResult.csv')
PcoolingPredictedValue = PcoolingPredicted.values
ITPredicted = pd.read_csv('ITpredictionResult.csv')
ITPredictedValue = ITPredicted.values
ConPDCbase = OPTmodel.addConstr(Pdc_base[i] == np.transpose(ITPredictedValue[i]) + np.transpose(PcoolingPredictedValue[i]))
OPTmodel.update()
# OPTmodel.addConstr(Pdc_base[i]==prediction[i])
OPTmodel.setObjective((np.transpose(Pdc_base[i])-float(DR[i]) - (Pdc[i]) ), GRB.MINIMIZE)
OPTmodel.update()
OPTmodel.optimize()
print Pdc_base[i].X
#print Ebess_i[i].X
#print Phvac_flex[i].X
print Tdc_i[i]
print Pdc[i]
print Phvac_flex[i]
print Pbess_flex[i]
print Pbess_out[i]
print Pbess_in[i]
print Ebess_i[i]
print Pbess_flex_[i]
print Phvac_down[i]
print Phvac_up[i]
'''
def get_results(self):
"""
This function gets the results of the current optimization model
Returns
-------
"""
HVACresult = np.zeros(1,N)
BatteryResult = np.zeros(1,N)
SOC = np.zeros(1,N)
#r_Q_dot = np.zeros((self.gp.N_H, self.N_S))
#r_P = np.zeros((self.gp.N_H, self.N_S))
#r_P_self = np.zeros((self.gp.N_H, self.N_S))
#r_P_ex = np.zeros((self.gp.N_H, self.N_S))
#r_Q_dot_gas = np.zeros((self.gp.N_H, self.N_S))
#Load = np.zeros((self.gp.N_H, self.N_S))
try:
for t in range(1,N):
HVACresult[t]= Phvac_flex[t].X
BatteryResult[t]=Pbess_flex[t].X
SOC[t] = Ebess_i[t].X / Ebess_max
except:
pass
return { 'SOC' : SOC , 'BatteryResult': BatteryResult }
print OPTmodel.getVars()
# get results
Temp = {}
Battery = {}
Ebess_result = {}
ITloadd = {}
for t in range(1,N):
Temp[t] = OPTmodel.getVarByName("Tdc_i" )
Battery[t] = OPTmodel.getVarByName("PBESS_flex" )
Ebess_result[t] = OPTmodel.getVarByName("Ebess_i" )
#r_P_e[t] = model.getVarByName("P_export_%s_0" % t).X
fig, axes = plt.subplots(4, 1)
# plot elctricity
ax5 = axes[2]
ax6 = ax5.twinx()
ax5.plot( [Temp[t] for t in range(1,N)], 'g-')
ax6.plot([Ebess_result[t] for t in range(1,N)], 'b-')
ax5.set_xlabel('Time index')
ax5.set_ylabel('Power Import [W]', color='g')
ax6.set_ylabel('Power CHP [W]', color='b')
ax7 = axes[3]
ax7.plot([Battery[t] for t in range(1,N)], 'g-')
ax7.set_ylabel('Power Export [W]', color='g')
'''
print Pflex_i.values()
# print OPTmodel.getVars()
print OPTmodel.feasibility()
print OPTmodel.getObjective()
print Pdc_base.values()
'''
b = map(float, Phvac_flex)
plt.plot(b)
plt.show()
'''
#c = map(float, Pbess_flex_)
#plt.plot(c)
#plt.show()
print OPTmodel
print Tdc_i.values()
# get results
print OPTmodel.getVars()
# print OPTmodel.getAttr('EBESS_i')
status = OPTmodel.status
print status
# print Con10,Con12
print Phvac_flex.values()
print Pbess_flex.values()
print Ebess_i.values()
print OPTmodel.objval
print Tdc_i
print Pbess_in
print Pbess_out.values()
# print Pbess_flex
# print Phvac_flex
# print Ebess_i
print Pflex_i.values()
print Pbess_flex_.values()
#print OPTmodel.getVars()
print OPTmodel.feasibility()
print OPTmodel.getObjective()
print Ebess_i.values()
if OPTmodel.status == GRB.Status.INF_OR_UNBD:
# Turn presolve off to determine whether model is infeasible
# or unbounded
OPTmodel.setParam(GRB.Param.Presolve, 0)
OPTmodel.optimize()
OPTmodel.write("mymodel.lp")
if OPTmodel.status == GRB.Status.OPTIMAL:
print('Optimal objective: %g' % OPTmodel.objVal)
OPTmodel.write('model.sol')
exit(0)
elif OPTmodel.status != GRB.Status.INFEASIBLE:
print('Optimization was stopped with status %d' % OPTmodel.status)
exit(0)
# Model is infeasible - compute an Irreducible Inconsistent Subsystem (IIS)
print('')
print('Model is infeasible')
OPTmodel.computeIIS()
OPTmodel.write("model.ilp")
print("IIS written to file 'model.ilp'")
I want to plot the computed values from gurobi but when I want to get the X attribute of gurobi variable it says that AttributeError: it has no attribute 'X' and the when I cast the value from float to int it just showed me the empty plot but at the lp file I could see the result of each iteration
I am anxiously waiting for your response
cherrs
This is for my python code
enter code herefileName = open(flename,"r")
dccount = 0
dcCycles = []
ccCycles = []
temp=0
ccCY = []
dcCY = []
for line in fileName:
length = len(line)
for i in range(length):
if i >=4:
x = line[i-1]
y = line[i]
bcValue = x+y
if bcValue == "IV":
ivval = float(int(line[i+2:i+7]))
if ivval > 0:
if temp == 1:
dcCY.append(dcCycles)
dcCycles=[]
cc = line[dccount:i-5]
ccCycles.append(cc)
dccount = i
temp =0
else:
cc = line[dccount:i-5]
ccCycles.append(cc)
else:
if temp == 0:
ccCY.append(ccCycles)
ccCycles=[]
dc = line[dccount:i-6]
dcCycles.append(dc)
dccount = i
temp = 1
else:
dc = line[dccount:i-6]
dcCycles.append(dc)
fileName.close()
in memory error got in cc = line[dccount:i-5] this ling how to resolve please help me
It looks like dccount was not yet defined at that point. You should have gotten a NameError