Develop Reference

Multi-Input Multi-Output in Genetic algorithm (python)

I wrote a GA program with python with 1 input, output and it works fine. But I want to find a solution with more input and output but I don't know how.
Example from https://pygad.readthedocs.io/:
Given function: y = f(w1:w6) = w1x1 + w2x2 + w3x3 + w4x4 + w5x5 + 6wx6
with
input(x1:x6)=(4,-2,3.5,5,-11,-4.7) and y=44
solution = (w1:w6)
But I want to find a solution with more input and output like input1 = (1,5,-3,5,-1,-4), y1 = 50.

Thanks for using PyGAD.
You can find the example you are looking for at this script. Here is the code:
import pygad
import numpy
"""
Given the following function:
y = f(w1:w6) = w1x1 + w2x2 + w3x3 + w4x4 + w5x5 + 6wx6
where (x1,x2,x3,x4,x5,x6)=(4,-2,3.5,5,-11,-4.7) and y=44
What are the best values for the 6 weights (w1 to w6)? We are going to use the genetic algorithm to optimize this function.
"""
function_inputs = [4,-2,3.5,5,-11,-4.7] # Function inputs.
desired_output = 44 # Function output.
def fitness_func(solution, solution_idx):
# Calculating the fitness value of each solution in the current population.
# The fitness function calulates the sum of products between each input and its corresponding weight.
output = numpy.sum(solution*function_inputs)
# The value 0.000001 is used to avoid the Inf value when the denominator numpy.abs(output - desired_output) is 0.0.
fitness = 1.0 / (numpy.abs(output - desired_output) + 0.000001)
return fitness
fitness_function = fitness_func
num_generations = 100 # Number of generations.
num_parents_mating = 10 # Number of solutions to be selected as parents in the mating pool.
# To prepare the initial population, there are 2 ways:
# 1) Prepare it yourself and pass it to the initial_population parameter. This way is useful when the user wants to start the genetic algorithm with a custom initial population.
# 2) Assign valid integer values to the sol_per_pop and num_genes parameters. If the initial_population parameter exists, then the sol_per_pop and num_genes parameters are useless.
sol_per_pop = 20 # Number of solutions in the population.
num_genes = len(function_inputs)
init_range_low = -2
init_range_high = 5
parent_selection_type = "sss" # Type of parent selection.
keep_parents = -1 # Number of parents to keep in the next population. -1 means keep all parents and 0 means keep nothing.
crossover_type = "single_point" # Type of the crossover operator.
# Parameters of the mutation operation.
mutation_type = "random" # Type of the mutation operator.
mutation_percent_genes = 10 # Percentage of genes to mutate. This parameter has no action if the parameter mutation_num_genes exists or when mutation_type is None.
last_fitness = 0
def callback_generation(ga_instance):
global last_fitness
print("Generation = {generation}".format(generation=ga_instance.generations_completed))
print("Fitness = {fitness}".format(fitness=ga_instance.best_solution()[1]))
print("Change = {change}".format(change=ga_instance.best_solution()[1] - last_fitness))
last_fitness = ga_instance.best_solution()[1]
# Creating an instance of the GA class inside the ga module. Some parameters are initialized within the constructor.
ga_instance = pygad.GA(num_generations=num_generations,
num_parents_mating=num_parents_mating,
fitness_func=fitness_function,
sol_per_pop=sol_per_pop,
num_genes=num_genes,
init_range_low=init_range_low,
init_range_high=init_range_high,
parent_selection_type=parent_selection_type,
keep_parents=keep_parents,
crossover_type=crossover_type,
mutation_type=mutation_type,
mutation_percent_genes=mutation_percent_genes,
on_generation=callback_generation)
# Running the GA to optimize the parameters of the function.
ga_instance.run()
# After the generations complete, some plots are showed that summarize the how the outputs/fitenss values evolve over generations.
ga_instance.plot_result()
# Returning the details of the best solution.
solution, solution_fitness, solution_idx = ga_instance.best_solution()
print("Parameters of the best solution : {solution}".format(solution=solution))
print("Fitness value of the best solution = {solution_fitness}".format(solution_fitness=solution_fitness))
print("Index of the best solution : {solution_idx}".format(solution_idx=solution_idx))
prediction = numpy.sum(numpy.array(function_inputs)*solution)
print("Predicted output based on the best solution : {prediction}".format(prediction=prediction))
if ga_instance.best_solution_generation != -1:
print("Best fitness value reached after {best_solution_generation} generations.".format(best_solution_generation=ga_instance.best_solution_generation))
If you have any questions, please let me know!

IB_insync returning tiny integer for shares (1 = 100, 0 = 50?) need float or proper scaled int

Using IB_insync API.
When loading ticker.Domticks and receiving the list of ticks, the dollar amount appears to be correct, but the shares show as small integers of 0,1,3,6 etc... When they should most likely be scaled 100x... and zero is most likely for less than 100 shares. Because its not a float it can not be scaled. Does anyone know why it would be returning the shares number incorrectly? I did recently subscribe to ASX australian exchange, and noticed that the shares number came back in the thousands, so it is presumably correct. contract = Stock('AAPL', "ISLAND","USD") > contract = Stock('CBA', "ASX","AUD")
def runner(ticker):
global elements
# print(ticker.domTicks)
for i in range(100):
if i < len(ticker.domTicks):
grab = ticker.domTicks[i]
elements.append(grab)
if __name__ == "__main__":
depth = 120
time_samples = 260
ib = IB()
ib.connect('127.0.0.1', 7497, clientId=2)
list_of_exchanges = ib.reqMktDepthExchanges()
for items in list_of_exchanges:
print(items)
print(list_of_exchanges)
contract = Stock('AAPL', "ISLAND","USD")
last_bid_book = np.zeros((0,depth))
print(last_bid_book)
last_ask_book = np.zeros((0,depth))
elements = []
ticker = ib.reqMktDepth(contract)
ib.sleep(1)
ticker.updateEvent += runner

Only round lots (and not odd lots) are typically returned with the top-of-book market data feed because the NBBO (National Best Bid/Best Offer) rules only pertain to round-lot orders.
What is an "Odd Lot" in stocks?
Odd-Lot orders are not posted to the bid/ask data on exchanges
As such the bid/ask data is returned with a multiplier that can be found in the mdSizeMultiplier field of the IBApi.ContractDetails class.

Mapping and iterating nested dictionaries

I am not too familiar with python but have a working understanding of the basics. I believe that I need dictionaries, but what I am currently doing is not working and likely very ineffective time-wise.
I am trying to create a cross matrix that links reviews between users given: the list of reviewers, their individual reviews, metadata related to the reviews.
NOTE : This is written in Python 2.7.10 - I cannot use Python 3 because outdated systems this will be run on, yada yada.
For initialization I have the following:
print '\nCompiling Review Maps... ';
LbidMap = {};
TbidMap = {};
for user in reviewer_idx :
for review in data['Reviewer Reviews'][user] :
reviewInfo = data['Review Information'][review];
stars = float(reviewInfo['stars']);
bid = reviewInfo['business_id'];
# Initialize lists where necessary
# !!!! I know this is probably not effective, but am unsure of
# a better method. Open to suggestions !!!!!
if bid not in LbidMap:
LbidMap[bid] = {};
TbidMap[bid] = {};
if stars not in LbidMap[bid] :
LbidMap[bid][stars] = {};
if user not in TbidMap[bid] :
TbidMap[bid][user] = {};
# Track information on ratings to each business
LbidMap[bid][stars][user] = review;
TbidMap[bid][user][review] = stars;
(where 'bid' is short for "Business ID", pos_list is an input given by user at runtime)
I then go on and try to create a mapping of users who gave a "positive" review to a business T who also gave business L a rating of X (e.g., 5 people rated business L 4/5 stars, how many of those people also gave a "positive" review to business T?)
For mapping I have the following:
# Determine and map all users who rated business L as rL
# and gave business T a positive rating
print '\nCross matching ratings across businesses';
cross_TrL = [];
for Tbid in TbidMap :
for Lbid in LbidMap :
# Ensure T and L aren't the same business
if Tbid != Lbid :
for stars in LbidMap[Lbid] :
starSum = len(LbidMap[Lbid][stars]);
posTbid = 0;
for user in LbidMap[Lbid][stars] :
if user in TbidMap[Tbid] :
rid = LbidMap[Lbid][stars][user];
print 'Tbid:%s Lbid:%s user:%s rid:%s'%(Tbid, Lbid, user, rid);
reviewRate = TbidMap[Tbid][user][rid];
# If true, then we have pos review for T from L
if reviewRate in pos_list :
posTbid += 1;
numerator = posTbid + 1;
denominator = starSum + 1;
probability = float(numerator) / denominator;
I currently receive the following error (print out of current vars also provided):
Tbid:OlpyplEJ_c_hFxyand_Wxw Lbid:W0eocyGliMbg8NScqERaiA user:Neal_1EVupQKZKv3NsC2DA rid:TAIDnnpBMR16BwZsap9uwA
Traceback (most recent call last):
File "run_edge_testAdvProb.py", line 90, in <module>
reviewRate = TbidMap[Tbid][user][rid];
KeyError: u'TAIDnnpBMR16BwZsap9uwA'
So, I know the KeyError is on what should be the rid (review ID) at that particular moment within TbidMap, however it seems to me that the Key was somehow not included within the first code block of initialization.
What am I doing wrong? Additionally, suggestions on how to improve clock cycles on the second code block is welcomed.
EDIT: I realized that I was trying to locate rid of Tbid using the rid from Lbid, however rid is unique to each review so you would not have a Tbid.rid == Lbid.rid.
Updated the second code block, as such:
cross_TrL = [];
for Tbid in TbidMap :
for Lbid in LbidMap :
# Ensure T and L aren't the same business
if Tbid != Lbid :
# Get numer of reviews at EACH STAR rate for L
for stars in LbidMap[Lbid] :
starSum = len(LbidMap[Lbid][stars]);
posTbid = 0;
# For each review check if user rated the Tbid
for Lreview in LbidMap[Lbid][stars] :
user = LbidMap[Lbid][stars][Lreview];
if user in TbidMap[Tbid] :
# user rev'd Tbid, get their Trid
# and see if they gave Tbid a pos rev
for Trid in TbidMap[Tbid][user] :
# Currently this does not account for multiple reviews
# given by the same person. Just want to get this
# working and then I'll minimize this
Tstar = TbidMap[Tbid][user][Trid];
print 'Tbid:%s Lbid:%s user:%s Trid:%s'%(Tbid, Lbid, user, Trid);
if Tstar in pos_list :
posTbid += 1;
numerator = posTbid + 1;
denominator = starSum + 1;
probability = float(numerator) / denominator;
evaluation = {'Tbid':Tbid, 'Lbid':Lbid, 'star':stars, 'prob':probability}
cross_TrL.append(evaluation);
Still slow, but I no longer receive the error.

How can I get a variable from main.py module to use in another parameters module?

I have been tried to improve my code, using a sensitivity analysis of my model. The model is agent based model to simulate economics in a country.
I need to variate 8 parameters, each one at least over 10 different values. So, I have been doing this "by hand" is a little bit complicate (because I do some mistakes) and take a lot of time.
My model is constructed by a sequence, I run the "my_control.py" that control the number of repetitions and (I hope will) control the parameters selection to test. This "my_control.py" call the "my_main.py" where is descripted the "orders", and, the "main.py" call the "generator.py" where create my population. And so on...
So, I develop the process in "R style" (beacuse I have a little bit more experience in R), using the "sys.argvs" from the three loops of "my_control.py". The first loop is to controll the number of regions, the second one is to select the parameter among my 8 parameters, and the third one will used to select the value inside a sequence created with the values selected as the second loop.
Theoretically it would work, but it's doesn't work.
So, thank's in advance.
# my_control.py module
__author__ = 'Isaque'
import os
for reg in range(3):
for prm_test in range(10):
for prm_to_run in range(10):
cmd = "python mainmodel.py "+ str(reg) +" "+ str(prm_test) +" "+ str(prm_to_run)
print cmd
os.system(cmd)
# my_parameters.py module
from __future__ import division
import sys
import pandas as pd
import numpy as np
# System constrain: number of households should be larger than number of families
# System constrain: number of members in a family is determined by the average ratio of inputs #agents #families
#CREATING THE PARAMETERS IN DEFAULT VALUES
# RUN PARAMETERS
TOTAL_DAYS = 5040
TOTAL_AGENTS = 1000
TOTAL_FAMILIES = 400
TOTAL_HOUSEHOLDS = 450
TOTAL_FIRMS = 110
ALPHA = .2
BETA = .9
QUANTITY_TO_CHANGE_PRICES = 10
PERCENTAGE_CHANGE_PRICES = 0.05
LABOUR_MARKET = 0.3
SIZE_MARKET = 10
CONSUMPTION_SATISFACTION = .1
PERCENTAGE_CHECK_NEW_LOCATION = 0.1
TAX_CONSUMPTION = 0.01
# AUTOMATED SELECTION OF PARAMETERS
# CREATING A REGIONS NUMBER OBJECT
def prm():
regions = pd.DataFrame({"NUMBER_REGIONS" : [1, 4, 7]})
# CREATING A OBJECT WITH ALL PARAMETERS VARIATION POSSIBLE
prm_parameters = pd.DataFrame({"Parameters":('SIZE_MARKET','ALPHA','BETA','QUANTITY_TO_CHANGE_PRICES', 'PERCENTAGE_CHANGE_PRICES','LABOUR_MARKET','CONSUMPTION_SATISFACTION','PERCENTAGE_CHECK_NEW_LOCATION','TAX_CONSUMPTION')})
# Parameters values
#"prm_start" is the lowest value to test that parameter in the model
#"prm_end" is the highest value to test that parameter in the model
#"prm_num_classes" is the number of classes for the intervall of each o test that parameter in the model
# Market Alpha Beta Qt. %Price Labour Sats. Loc. Tax
prm_start = pd.DataFrame({"Start" : (1, 0.1, 0.1, 10, 0.01, 0.1, 0.01, 0.01, 0.1)})
prm_end = pd.DataFrame({"End" : (50, 1, 1, 1000, 1, 1, 1, 1, 1)})
prm_num_classes = pd.DataFrame({"N_classes": (10, 10, 10, 10, 10, 10, 10, 10, 10)})
parameters_to_test = pd.concat([prm_parameters, prm_start,prm_end,prm_num_classes], axis=1)
# CALLING THE ARGV FROM LOOPS TO SELECT THE VALUES OF PARAMETERS IN MY OBJECT "parameters_to_test"
# "reg_index" will be used to test the different number of regions in the model
reg_index = sys.argv[1]
# "prm_test" and "prm_to_run" will be used to select the parameters values inside of "parameters_to_test" object
prm_test = sys.argv[2]
prm_to_run = sys.argv[3]
#renaming the variable for "NUMBER_REGIONS" selected to test
globals()["NUMBER_REGIONS"] = int(regions.iloc[reg_index])
# Selecting the row (each row represent one parameter)
prm_run = parameters_to_test.iloc[prm_test]
# Creating a variable length to generate the sequence of parameters
interval_out = prm_run['End']/prm_run['N_classes']
# Creating the sequence with all possible values to test
seq_prm_run = pd.DataFrame(np.arange(prm_run['Start'],(prm_run['End']+interval_out),interval_out))
# Selecting the name of variable from the selected row that create the sequence
name_prm = str(prm_run['Parameters'])
#Selecting the value inside the sequence and renaming the variable as the name
globals()[name_prm] = (seq_prm_run.iloc[prm_to_run])[0]
print name_prm
print (seq_prm_run.iloc[prm_to_run])[0]
if __name__ == '__main__':
prm()

Rename a persistent object within a definition (python)

Minimal Question:
def smooth(indicator, aggregation, tick):
storage.ZZZ = []
storage.ZZZZ = []
is the pertinent part of my definition, when I call that definition I'm using:
MA_now_smooth = smooth(MA, IN, I)[-1]
where MA is an input array, IN and I are constants; the definition is further defined below but ultimately returns the last input to storage.ZZZZ. What I want is to create custom storage objects that are named according to the "indicator" input so that the persistent variables don't overlap when calling upon this definition for myriad array inputs.
ie
smooth(MA, IN, I)[-1]
should create:
storage.ZZZ_MA
storage.ZZZZ_MA
but
smooth(MA2, IN, I)[-1]
should create:
storage.ZZZ_MA2
storage.ZZZZ_MA2
In Depth Question:
I'm creating an Simple Moving Average smoothing definition for TA-lib indicators at tradewave.net; TA-lib is a library of black box functions that give "Financial Technical Analysis" array outputs for things like "moving average" "exponential moving average" "stochastic" etc. My definition is a secondary simple smoothing of these TA-lib functions.
I'm having to do this because when "aggregating" candles counting backwards from current, I'm getting "wiggly" outputs; you can read more about that here if you need background: https://discuss.tradewave.net/t/aggregating-candles-some-thoughts
My definition code works well to create a list of smoothed values when smoothing a single indicator "MA"; a TA-lib array:
import talib
def smooth(indicator, aggregation, tick):
import math
A = int(math.ceil(aggregation/tick))
if info.tick == 0:
storage.ZZZ = []
storage.ZZZZ = []
storage.ZZZ.append(indicator[-1])
storage.ZZZ = storage.ZZZ[-A:]
ZZZ = sum(storage.ZZZ) / len(storage.ZZZ)
storage.ZZZZ.append(ZZZ)
storage.ZZZZ = storage.ZZZZ[-250:]
return storage.ZZZZ
def tick():
I = info.interval
period = 10
IN = 3600
instrument = pairs.btc_usd
C = data(interval=IN)[instrument].warmup_period('close')
MA = talib.MA(C, timeperiod=period, matype=0)
MA_now = MA[-1]
MA_now_smooth = smooth(MA, IN, I)[-1]
plot('MA', MA_now)
plot('MA_smooth', MA_now_smooth)
However, when I attempt to smooth more than one indicator with the same definition, it fails because the persistent variables in the definition are the same for both MA and MA2. This does not work:
import talib
def smooth(indicator, aggregation, tick):
import math
A = int(math.ceil(aggregation/tick))
if info.tick == 0:
storage.ZZZ = []
storage.ZZZZ = []
storage.ZZZ.append(indicator[-1])
storage.ZZZ = storage.ZZZ[-A:]
ZZZ = sum(storage.ZZZ) / len(storage.ZZZ)
storage.ZZZZ.append(ZZZ)
storage.ZZZZ = storage.ZZZZ[-250:]
return storage.ZZZZ
def tick():
I = info.interval
period = 10
IN = 3600
instrument = pairs.btc_usd
C = data(interval=IN)[instrument].warmup_period('close')
MA = talib.MA(C, timeperiod=period, matype=0)
MA2 = talib.MA(C, timeperiod=2*period, matype=0)
MA_now = MA[-1]
MA2_now = MA2[-1]
MA_now_smooth = smooth(MA, IN, I)[-1]
MA2_now_smooth = smooth(MA2, IN, I)[-1]
plot('MA', MA_now)
plot('MA2', MA2_now)
plot('MA_smooth', MA_now_smooth)
plot('MA2_smooth', MA2_now_smooth)
What I would like to do... and don't understand how to do:
I'd like for the definition to create a new persistent storage object for each new input and I'd like for the names of my objects to detect the name of the "indicator" input, ie:
storage.ZZZ_MA
storage.ZZZZ_MA
ZZZ_MA
for the "MA" smoothing and
storage.ZZZ_MA2
storage.ZZZZ_MA2
ZZZ_MA2
for "MA2" smoothing
I would like to be able to reuse this definition with many different array inputs for "indicator" and for each instance use the name of the indicator array appended to the persistent object names used in the definition. For example:
storage.ZZZ_MA3
storage.ZZZ_MA4
etc.
In the instances below info.interval is my tick size of 15 minutes (900 sec) and my aggregation was 1 hour (3600 sec)
With the single output of "MA" and correct smoothing
With dual outputs of "MA" and "MA2" I'm getting incorrect smoothing
In the second image I'm looking for a two "smooth" lines one in the middle of the wiggly red plot and the other in the middle of wiggly blue plot. Instead I'm getting two identical wiggly lines (purple & orange) that split the difference. I understand why, but I don't know how to fix it.
1) please show me how
2) please tell me what I'm looking to do is "called" and point me to some tags/posts where I can learn more.
Thanks for your help!
LP

Make storage a dict, and use string keys rather than trying to create and access dynamic variables?

Well I've arrived at an interim solution.
While I like this solution as its doing everything I need. I would like to eliminate the redundant "label" input. Is there any way for me to reference the name of my input parameter/argument "indicator" instead of its object so that I could return to my original 3 input parameters rather than 4?
I tried this:
def smooth(indicator, aggregation, tick):
import math
A = int(math.ceil(aggregation/tick))
ZZZ = 'ZZZ_%s' % dict([(t.__name__, t) for t in indicator])
ZZZZ = 'ZZZZ_%s' % dict([(t.__name__, t) for t in indicator])
if info.tick == 0:
storage[ZZZ] = []
storage[ZZZZ] = []
storage[ZZZ].append(indicator[-1])
storage[ZZZ] = storage[ZZZ][-A:]
ZZZZZ = sum(storage[ZZZ]) / len(storage[ZZZ])
storage[ZZZZ].append(ZZZZZ)
storage[ZZZZ] = storage[ZZZZ][-250:]
return storage[ZZZZ]
but I get:
File "", line 259, in File "", line 31, in tick File "", line 6, in smooth AttributeError: 'numpy.float64' object has no attribute 'name'
Here is my current 4 argument definition smoothing 4 different TA-lib moving averages. This same definition can be used with many other aggregated TA-lib indicators. It should work with ANY aggregate/tick size ratio including 1:1.
import talib
def smooth(indicator, aggregation, tick, label):
import math
A = int(math.ceil(aggregation/tick))
ZZZ = 'ZZZ_%s' % label
ZZZZ = 'ZZZZ_%s' % label
if info.tick == 0:
storage[ZZZ] = []
storage[ZZZZ] = []
storage[ZZZ].append(indicator[-1])
storage[ZZZ] = storage[ZZZ][-A:]
ZZZZZ = sum(storage[ZZZ]) / len(storage[ZZZ])
storage[ZZZZ].append(ZZZZZ)
storage[ZZZZ] = storage[ZZZZ][-250:]
return storage[ZZZZ]
def tick():
I = info.interval
period = 10
IN = 3600
instrument = pairs.btc_usd
C = data(interval=IN)[instrument].warmup_period('close')
MA1 = talib.MA(C, timeperiod=period, matype=0)
MA2 = talib.MA(C, timeperiod=2*period, matype=0)
MA3 = talib.MA(C, timeperiod=3*period, matype=0)
MA4 = talib.MA(C, timeperiod=4*period, matype=0)
MA1_now = MA1[-1]
MA2_now = MA2[-1]
MA3_now = MA3[-1]
MA4_now = MA4[-1]
MA1_now_smooth = smooth(MA1, IN, I, 'MA1')[-1]
MA2_now_smooth = smooth(MA2, IN, I, 'MA2')[-1]
MA3_now_smooth = smooth(MA3, IN, I, 'MA3')[-1]
MA4_now_smooth = smooth(MA4, IN, I, 'MA4')[-1]
plot('MA1', MA1_now)
plot('MA2', MA2_now)
plot('MA3', MA3_now)
plot('MA4', MA4_now)
plot('MA1_smooth', MA1_now_smooth)
plot('MA2_smooth', MA2_now_smooth)
plot('MA3_smooth', MA3_now_smooth)
plot('MA4_smooth', MA4_now_smooth)
h/t james for collaboration