I am trying to implement in my simulation this code:
https://numpy.org/doc/stable/reference/random/multithreading.html
but I can't work it out.
If I follow the example in the link, I get
mrng = MultithreadedRNG(10000000, seed=0)
mrng.fill()
print(mrng.values[-1])
> 0.0
and all the other values are 0 too.
If I give a smaller input number, as 40, I get
mrng = MultithreadedRNG(40)
mrng.fill()
print(mrng.values[-1])
> array([1.08305179e-311, 1.08304781e-311, 1.36362118e-321, nan,
6.95195359e-310, ...., 7.27916164e-095, 3.81693953e+180])
What am I doing wrong? I just would like to implement this multiprocessing code to a random bits (0 / 1) generator.
There is a bug in the example, I believe. You have to wrap PCG64 into Generator interface.
Try code below
class MultithreadedRNG(object):
def __init__(self, n, seed=None, threads=None):
rg = PCG64(seed)
if threads is None:
threads = multiprocessing.cpu_count()
self.threads = threads
self._random_generators = [Generator(rg)]
last_rg = rg
for _ in range(0, threads-1):
new_rg = last_rg.jumped()
self._random_generators.append(Generator(new_rg))
last_rg = new_rg
self.n = n
self.executor = concurrent.futures.ThreadPoolExecutor(threads)
self.values = np.empty(n)
self.step = np.ceil(n / threads).astype(np.int_)
def fill(self):
def _fill(gen, out, first, last):
gen.standard_normal(out=out[first:last])
futures = {}
for i in range(self.threads):
args = (_fill,
self._random_generators[i],
self.values,
i * self.step,
(i + 1) * self.step)
futures[self.executor.submit(*args)] = i
concurrent.futures.wait(futures)
def __del__(self):
self.executor.shutdown(False)
Didn't test it much, but values looks ok
Related
I have a app with pyqt5 and receive data from udp, pack them and show.
I use threading(python built-in thread) for my app(one thread for dl_data, one thread for save_log and one thread for ui) but in real time I have a lots of lag in showing final image.
Part of my code for pack data is:
self.data_log = [[0 for i in range(1000)] for j in range(1600)]
def unpack(self,pack):
tmp = bytearray()
for i in range(3):
tmp.extend(pack[i])
return np.asarray([newdiv(struct.unpack('<H',tmp[x+1:x+3])[0],256) for x in range(0,(self.NP) * 4,4)])
def merge(self,data):
data = self.unpack(data)
data = np.roll(data,200,0)
self.data_log.extend([data])
def dl_data(self):
while 1:
for i in range(self.NPBN * 3):
tmp = self.recv(self.bytes)
self.merge(tmp)
def process_feed(self):
while True:
data = self.tcpsocket.recv(1024)
if b'pressed' in data :
if not self.get_data.is_alive():
self.get_data.start()
def show(self):
.....
if self.Mode == 'run':
pressed= threading.Thread(target=self.process_feed)
pressed.start()
get_data = threading.Thread(target=self.dl_data)
newdiv function is "Making division in Python faster" link
This is the code. I think the solver could be glpk instead of gurobi. I got the error before it tries to solve the problem.
from pyomo.environ import *
from pyomo.opt import SolverFactory, SolverStatus
PrintSolverOutput = False ###
model = ConcreteModel()
model.dual = Suffix(direction =Suffix.IMPORT)
model.X = Var(I,T, within = NonNegativeReals)
model.In = Var(I,T, within = NonNegativeReals)
model.S = Var(T, within = NonNegativeReals)
def Objetivo(model):
return (sum(C[i]*model.X[i,t]*((1+tau[i])**(t-1))+Ch[i]*model.In[i,t]
for i in I for t in T)
+sum(Cs*model.S[t]
for t in T)
)
model.Total_Cost = Objective(rule = Objetivo, sense= minimize)
def Balance(model,i,t):
if t==1:
return model.X[i,t]+I0[i]-model.In[i,t]==D[i,t]
elif t>=2:
return model.X[i,t]+model.IN[i,t-1]-model.In[i,t]==D[i,t]
def Horas(model, t):
return sum(r[i]*model.X[i,t] for i in I) <= H+model.S[t]
def Limite(model,t):
return model.S[T]<=LS
model.RBalance = Constraint(I,T,rule=Balance)
model.RHoras = Constraint(T,rule=Horas)
model.RLimiteoras = Constraint(T,rule=Limite)
opt = SolverFactory("gurobi")
This is the data
I forgot to put the data before.
T = [1,2,3,4,5,6]
I = ['A','B']
D = {('A',1):300,
('A',2):400,
('A',3):500,
('A',4):600,
('A',5):800,
('A',6):700,
('B',1):700,
('B',2):600,
('B',3):500,
('B',4):400,
('B',5):300,
('B',6):400}
tau = {'A':0.02,'B':0.01}
r = {'A':5,'B':3}
H = 3520
LS = 800
C = {'A':150,'B':120}
Ch = {'A':8,'B':4}
Cs = 6
I0 = {'A':100,'B':250}
error
The code is from a youtube tutorial and it worked for him but not for me. Why?
Aside from fixing two typos in your code, this model computes and solves for me without that error. Make these fixes, run it again, re-post the exact error with line number and the exact code that produces the error if stuck...
I am trying to run a NEAT algorithm using this python implementation. This is the original file from the library that is relevant for my question:
from neat.graphs import feed_forward_layers
class FeedForwardNetwork(object):
def __init__(self, inputs, outputs, node_evals):
self.input_nodes = inputs
self.output_nodes = outputs
self.node_evals = node_evals
self.values = dict((key, 0.0) for key in inputs + outputs)
def activate(self, inputs):
if len(self.input_nodes) != len(inputs):
raise RuntimeError("Expected {0:n} inputs, got {1:n}".format(len(self.input_nodes), len(inputs)))
for k, v in zip(self.input_nodes, inputs):
self.values[k] = v
for node, act_func, agg_func, bias, response, links in self.node_evals:
node_inputs = []
for i, w in links:
node_inputs.append(self.values[i] * w)
s = agg_func(node_inputs)
self.values[node] = act_func(bias + response * s)
return [self.values[i] for i in self.output_nodes]
#staticmethod
def create(genome, config):
""" Receives a genome and returns its phenotype (a FeedForwardNetwork). """
# Gather expressed connections.
connections = [cg.key for cg in genome.connections.values() if cg.enabled]
layers = feed_forward_layers(config.genome_config.input_keys, config.genome_config.output_keys, connections)
node_evals = []
for layer in layers:
for node in layer:
inputs = []
node_expr = [] # currently unused
for conn_key in connections:
inode, onode = conn_key
if onode == node:
cg = genome.connections[conn_key]
inputs.append((inode, cg.weight))
node_expr.append("v[{}] * {:.7e}".format(inode, cg.weight))
ng = genome.nodes[node]
aggregation_function = config.genome_config.aggregation_function_defs.get(ng.aggregation)
activation_function = config.genome_config.activation_defs.get(ng.activation)
node_evals.append((node, activation_function, aggregation_function, ng.bias, ng.response, inputs))
return FeedForwardNetwork(config.genome_config.input_keys, config.genome_config.output_keys, node_evals)
Since I evaluate the performance of my neural networks on a large dataset, I wanted to speed up the activate method using numba jit. In order to not fall back into numbas object mode I had to update the implementation of the activate method (and hence also the fields of the FeedForwardNetwork class) using only datatypes supported by numba. This is what I came up with (create is the same as before):
from neat.graphs import feed_forward_layers
from neat.six_util import itervalues
import numba
from numba import jit, njit
from numba.typed import List, Dict
import numpy as np
import math
#jit(nopython=True)
def activate(input_nodes, output_nodes, node_evals_node, node_evals_bias, node_evals_resp, node_evals_ins_nodes, node_evals_ins_conns, values, inputs):
for i in range(input_nodes.size):
values[input_nodes[i]] = inputs[i]
for node in range(len(node_evals_node)):
s = 0
for pred in range(len(node_evals_ins_nodes[node])):
s += values[node_evals_ins_nodes[node][pred]] * node_evals_ins_conns[node][pred]
values[node_evals_node[node]] = math.tanh(node_evals_bias[node] + node_evals_resp[node] * s)
return [values[output_nodes[i]] for i in range(output_nodes.size)]
class FeedForwardNetwork(object):
def __init__(self, inputs, outputs, node_evals):
self.input_nodes = np.array(inputs)
self.output_nodes = np.array(outputs)
# NODE_EVALS decomposition
self.node_evals_node = np.reshape(np.array(node_evals)[:, 0:1], (len(node_evals),)).astype(np.int64)
self.node_evals_bias = np.reshape(np.array(node_evals)[:, 3:4], (len(node_evals),)).astype(np.float64)
self.node_evals_resp = np.reshape(np.array(node_evals)[:, 4:5], (len(node_evals),)).astype(np.float64)
temp = np.array(node_evals)[:, 5:6]
self.node_evals_ins_nodes = List()
self.node_evals_ins_conns = List()
for node in range(temp.size):
l = List()
m = List()
for predecessor in range(len(temp[node])):
l.append(temp[0][node][predecessor][0])
m.append(temp[0][node][predecessor][1])
self.node_evals_ins_nodes.append(l)
self.node_evals_ins_conns.append(m)
self.values = Dict()
# Set types of dict
self.values[0] = float(1)
self.values.pop(0)
This is the code I call the create and activate method in:
def eval_single_genome(genome, config, thread_id, result):
net = neat.nn.FeedForwardNetwork.create(genome, config)
error_sum = 0
for i, row in PRICES.iterrows():
prediction = feed_forward.activate(net.input_nodes, net.output_nodes, net.node_evals_node, net.node_evals_bias, net.node_evals_resp, net.node_evals_ins_nodes, net.node_evals_ins_conns, net.values, np.array([0]))
error_sum += (prediction - PRICES.iloc[i]['open']) ** 2
result[thread_id] = error_sum
The code compiles and runs without errors or warnings which (as far as I've understood) indicates that numba should be able to optimize my implementation. But adding/removing the #jit(nopython=True)decorator doesn't change the runtime at all.
Did I overlook something? Or is there just nothing that numba can improve in my case?
def compute_linear_gradient_manually(args,model,data_tr,data_te,device):
N_tr = len(data_tr)
N_te = len(data_te)
kernel_tr = torch.zeros(N_tr,N_tr).to(device)
kernel_te = torch.zeros(N_te,N_tr).to(device)
for i in range(N_tr):
grad_tr = compute_layer_gradient(args,model,data_tr[i])
for j in range(N_tr):
grad_tr_ = compute_layer_gradient(args,model,data_tr[j])
kernel_tr[i][j] = torch.mm(grad_tr,grad_tr_.t())
for k in range(N_te):
grad_tr_ = compute_layer_gradient(args,model,data_te[k])
kernel_te[i][k] = torch.mm(grad_tr,grad_tr_.t())
return kernel_tr,kernel_te
def compute_layer_gradient(args,model,data):
p = ((torch.mm(data.view(-1,3072),model.weight1)+model.bias1)>0).float()
weight2_grad = ((torch.mm(data.view(-1,3072),model.weight1)+model.bias1)*p).t()
weight1_grad = torch.mm(data.view(-1,3072).t(),model.weight2.t()*p)
bias1_grad = math.sqrt(1/width)*model.weight2.t()*p.view(-1,width)
grad = torch.cat([weight1_grad.view(-1,3072*width),bias1_grad.view(-1,width),weight2_grad.view(-1,width)],dim=1)
return grad
I am using CUDA to compute two matrices kernel_tr and kernel_te. The function compute_layer_gradient returns a 1*m vector and data_tr and data_te are two matrices.
But the memory keeps increasing and I have tried using del grad_tr_ or something like that but it doesn't work.
How to correctly release the memory in this loop?
I've been working on a project that is a calculator for an electronic part. This is my first full-scale Python project and it has gone through many iterations along the way.
The data is not organized as well as I would like, which I think is making it difficult to improve the program.
I have a class for the module, which has two parts inside of it. Each part has its own attributes that are independent of each other, and the whole module itself has some attributes that the parts don't care about, and some attributes that the parts need. The module also has to do some calculations based on the results of the independent calculations done inside the parts. Here is a photo of the idea I'm trying to explain. As an aside, sometimes there are modules with only one part, but I can suppress the other part by applying some 0's arrays. I would like to be able to have a module where the one part is entirely missing, but that is not my current goal.
The problem is that my class has ~100 lines of self.XXX = None at the beginning to initialize everything, and several functions which IMO are repetitive. It is also quite difficult to traverse the data when stepping through the code - for example, I have to find variables like self.current__partA and self.current__partB. What I think would be helpful is to have something like self.partA.current. If it is done this way, I think it would be more readable.
The problem is, I tried subclasses and it seems like I can't achieve this kind of idea because when I initialize the subclass, I have to initalize a new superclass, meaning that there are two superclasses (two modules with a total of 4 parts, when I want 1 module/2 parts), so I can't really access the info of both subclasses from the superclass because each subclass will have its own instance of the superclass.
I also looked at inner classes but there is an issue where I don't think I can truly access the outer class from the inner class, which kind of defeats the purpose of using this. That could work to a degree, but it would make my code longer and less readable, from what I am seeing.
The first solutions I had were things like dictionaries, which I don't totally hate, but that lead to really janky code that had very little tolerance for errors. The reason is, when you add a list to a dictionary, you can't have a function that automatically throws an error. I can check the dictionary, but it just feels unnatural. It seems to me that it would make more sense to keep each value as a class variable and use functions, getters and setters to manipulate it through the calculation.
My main goal is to organize the data and code effectively so that I am using less lines of code and the program is easier to modify, and it is easier to step through the process. I am not entirely sold on the class structure, it just seemed to be the best way to accommodate what I am trying to do. Is there a way to achieve what I am asking here, or is there a generally more pythonic way to organize my code that would result in a more effective solution?
class Module:
def __init__(self, module_file):
temp_ic, temp_value = self.get__corrected_value(module_file)
temp_if, temp_vf = self.get__corrected_value(module_file)
self.ic_value = interp1d(temp_ic, temp_value, fill_value='extrapolate')
self.ic_esw_on = interp1d(self.get__corrected_esw(module_file), self.get__corrected_esw(module_file["ESWON - IC ESWON"]), fill_value='extrapolate')
self.ic_esw_off = interp1d(self.get__corrected_esw(module_file["IC - IC ESWOFF"]), self.get__corrected_esw(module_file["ESWOFF - IC ESWOFF"]), fill_value='extrapolate')
self.rg_on_esw_on = interp1d(module_file["RGON - ESWON RGON"], module_file["ESWON - ESWON RGON"], fill_value='extrapolate')
self.rg_off_esw_off = interp1d(module_file["RGOFF - ESWOFF RGOFF"], module_file["ESWOFF - ESWOFF RGOFF"], fill_value='extrapolate')
self.ic_err = interp1d(self.get__corrected_esw(module_file["IC - IC ERR"]), self.get__corrected_esw(module_file["ERR - IC ERR"]), fill_value='extrapolate')
self.if_vf = interp1d(temp_if, temp_vf, fill_value='extrapolate')
self.rg_on_err = interp1d(module_file["RGON - ERR RGON"], module_file["ERR - ERR RGON"], fill_value='extrapolate')
self.nameplate_vcc = module_file['Nameplate VCC']
if module_file['vcc_ratio'] > 0:
self.vcc_ratio = module_file['vcc_ratio']
else:
self.vcc_ratio = 0
self.name = self.get__module_name(module_file)
self.current__PartA = []
self.current__PartB = []
self.some_thing_loss__PartA = []
self.esw_on_loss = []
self.esw_off_loss = []
self.esw_loss__PartA = []
self.energy__PartA = []
self.value__PartA = []
self.some_thing_loss__PartB = []
self.err_loss = []
self.energy__PartB = []
self.value__PartB = []
self.rg_scalar_esw_on = None
self.rg_scalar_esw_off = None
self.rg_scalar_err = None
self.value_dc__PartA = module_file['PartA value DC']
self.value_dc__PartB = module_file['PartB value DC']
self.value_dc__module = module_file['Module value DC']
self.trans_r_values__PartA = module_file["PartA R Values"]
self.trans_t_values__PartA = module_file["PartA T Values"]
self.trans_r_values__PartB = module_file["PartB R Values"]
self.trans_t_values__PartB = module_file["PartB T Values"]
self.some_thing_loss_total__PartA = None
self.some_thing_loss_total__PartB = None
self.esw_on_loss_total = None
self.esw_off_loss_total = None
self.esw_loss_total = None
self.err_loss_total = None
self.device_loss_total__PartA = None
self.device_loss_total__PartB = None
self.module_loss_total = None
self.delta_tcase_ave = None
self.delta_value_ave__PartA = None
self.delta_value_ave__PartB = None
self.nominal_value_ave__PartA = None
self.nominal_value_ave__PartB = None
self.delta_value_max__PartA = None
self.delta_value_max__PartB = None
self.nominal_value_max__PartA = None
self.nominal_value_max__PartB = None
self.value_max_PartA_list = []
self.value_max_PartB_list = []
self.thermal_interp_is_four_degree = self.check__thermal_interp()
self.switches_per_degree = None
self.input_output_freq = None
self.time_division = None
self.input_t_sink = None
self.step_size = None
self.step_range = None
self.sec_per_cycle_degree = None
self.duty_p = None
self.value_PartA_list = None
self.value_PartB_list = None
self.time_list = None
self.rad_list = None
self.value_max__PartA_thermo = None
self.value_max__PartB_thermo = None
self.value_max__time_value = None
def check__some_input_conditions_and_change_input(self): # todo could this be cleaned?
blah
def get__max_current(self):
return max(self.nominal_value_max__PartB, self.nominal_value_max__PartA)
def set__some_module_values(self, is_three_level, system): # todo call this something different, and break it out for 3-level
blah
def set_values_for_both_parts(self, input_instance, system_instance, module_location=None):
lots of blah
def set__current_PartA(self, current):
self.current__PartA = current
def set__current_partB(self, current):
blah
def calculate__another_other_loss_for_part_A(self, duty):
blah
def calculate__another_loss_for_partB(self, duty):
blah
def calculate__another_loss_for_partA(self, duty=None):
blah
def calculate__some_loss_for_partA(self, duty=None):
blah
def calculate__some_loss_for_partB(self, duty=None):
blah
def calculate__energy_power_for_both_parts(self):
blah
def calculate__temperatures_for_both_parts(self):
blah
def calculate__max_temp(self): # maybe split into PartA and PartB separately?
self.create_thermal_resistance_dict()
value_PartA_list = []
value_PartB_list = []
next_array_PartA = self.value__PartA
next_array_PartA = self.rotate(next_array_PartA, -1)
delta_p_PartA = [next_el - last_el for next_el, last_el in zip(next_array_PartA, self.value__PartA)]
last_power_PartA = self.value__PartA[-1] - self.device_loss_total__PartA
first_power_PartA = self.value__PartA[0] - self.device_loss_total__PartA
value_dict_PartA_added = [self.get_PartA_value_from_time(i * self.sec_per_cycle_degree + self.value_max__time_value) for i in range(self.step_range)]
value_dict_PartA_added = [old + new for old, new in zip(self.value_max__PartA_thermo, value_dict_PartA_added)]
value_PartA_inst_init = [self.input_t_sink + self.delta_value_ave__PartA + self.delta_tcase_ave - last_power_PartA * self.value_max__PartA_thermo[i] + first_power_PartA * value_dict_PartA_added[i] for i in range(self.step_range)]
delta_value_PartB = self.device_loss_total__PartB * self.value_dc__PartB
next_array_PartB = self.value__PartB
next_array_PartB = self.rotate(next_array_PartB, -1)
delta_p_PartB = [next_el - last_el for next_el, last_el in zip(next_array_PartB, self.value__PartB)]
last_power_PartB = self.value__PartB[-1] - self.device_loss_total__PartB
first_power_PartB = self.value__PartB[0] - self.device_loss_total__PartB
value_dict_PartB_added = [self.get_PartB_value_from_time(i * self.sec_per_cycle_degree + self.value_max__time_value) for i in range(self.step_range)]
value_dict_PartB_added = [old + new for old, new in zip(self.value_max__PartB_thermo, value_dict_PartB_added)]
value_PartB_inst_init = [self.input_t_sink + delta_value_PartB + self.delta_tcase_ave - last_power_PartB * self.value_max__PartB_thermo[i] + first_power_PartB * value_dict_PartB_added[i] for i in range(self.step_range)]
for index in range(self.step_range):
value_dict_PartA_fix = [value_dict_PartA_added[i] if i <= index else self.value_max__PartA_thermo[i] for i in range(self.step_range)]
# value_dict_PartA_fix_orig = [val for val in value_dict_PartA_fix]
value_dict_PartA_fix.reverse()
new_value_PartA = self.rotate(value_dict_PartA_fix, index)
new_value_PartA = new_value_PartA[:359]
temp_add_vals_PartA = [delta_p * value for delta_p, value in zip(delta_p_PartA, new_value_PartA)]
sum_temp_add_vals_PartA = sum(temp_add_vals_PartA)
value_PartA_list.append(sum_temp_add_vals_PartA)
value_dict_PartB_fix = [value_dict_PartB_added[i] if i <= index else self.value_max__PartB_thermo[i] for i in range(self.step_range)]
# value_dict_PartB_fix_orig = [val for val in value_dict_PartB_fix]
value_dict_PartB_fix.reverse()
new_value_PartB = self.rotate(value_dict_PartB_fix, index)
new_value_PartB = new_value_PartB[:359]
temp_add_vals_PartB = [delta_p * value for delta_p, value in zip(delta_p_PartB, new_value_PartB)]
sum_temp_add_vals_PartB = sum(temp_add_vals_PartB)
value_PartB_list.append(sum_temp_add_vals_PartB)
value_PartA_list = [value + diff for value, diff in zip(value_PartA_inst_init, value_PartA_list)]
value_ave_PartA = self.nominal_value_ave__PartA - np.average(value_PartA_list)
self.value_PartA_list = [value + value_ave_PartA for value in value_PartA_list]
value_PartB_list = [value + diff for value, diff in zip(value_PartB_inst_init, value_PartB_list)]
value_ave_PartB = self.nominal_value_ave__PartB - np.average(value_PartB_list)
self.value_PartB_list = [value + value_ave_PartB for value in value_PartB_list]
self.time_list = [i * self.sec_per_cycle_degree + self.value_max__time_value for i in range(self.step_range)]
self.rad_list = [i * self.step_size for i in range(self.step_range)]
self.nominal_value_max__PartA = max(value_PartA_list)
self.nominal_value_max__PartB = max(value_PartB_list)
self.delta_value_max__PartA = max(self.value_PartA_list) - self.input_t_sink
self.delta_value_max__PartB = max(self.value_PartB_list) - self.input_t_sink
self.value_max_PartA_list = value_PartA_list
self.value_max_PartB_list = value_PartB_list
def rotate(self, l, n):
return l[-n:] + l[:-n]
def do_calculation_for_either_part(self, step, spcd, index, scalar, growth, time): # todo does everything need to be passed in?
blah
def get_other_part's_value(self, time): # todo could this be folded into below
blah
def get_one_part's_value(self, time):
blah
def integrate_value_for_other_part(self, step, spcd, start_time, index): # todo could this be folded into below
blah
def integrate_value_for_one_part(self, step, spcd, start_time, index): # todo remove interp check
blah
def create_some_dict_for_both_parts(self): # todo could this be cleaned
50 lines of blah
def get__other_corrected_array(self, array): # todo could this be simplified?
blah
def get__corrected_array(self, input arrays): # todo is this necessary
blah
def get__some_value(self, value): # todo isn't there one of these already?
blah
def get__module_name(self, module_file):
blah
The commentators are correct that an MCVE would definitely enhance your post and so my answer is a bit limited. I just want to point out that your data members can be any python object.
So if your data access pattern would benefit from storing your data in pandas and interacting with it as pandas:
class YourClass:
def __init__(self, data):
self.data = # Your pandas df
Or json:
import json
class YourClass:
def __init__(self, data):
self.data = json.loads(data)
Or numpy:
class YourClass:
def __init__(self, data):
self.data = # Your numpy ndarray
And then your class can be called simply as YourClass(data)
Edit: Looking at your code, literally ALL of your self.value = None lines are superfluous in my view. if they are a members of a tabular data input they can be initialized:
class Module:
def __init__(self, data):
self.data = pd.DataFrame()
Once they are initialized as an empty dataframe their CRUD operations can map to the very mature pandas CRUD operations. Similarly self.data = {} for key-value pairs data structure like JSON, and so on. For the rest, you can catch the case where data.key is undefined in generic getters and setters and not bother with initializing them.