I have this model:
class Image(models.Model):
title = models.CharField(max_length=200)
image = models.ImageField(upload_to='img/')
signature = models.TextField(null = True)
The signature is a numpy monodimensional vector encoded in json. In order to make my query, I have to decode each object signature into a nparray, and make a dot product between each object's signature and a given vector, then annotate as a float (named "score") field beside each raw. Lastly I have to order from max to min.
I tried this in view.py
def image_sorted(request):
query_signature = extract_feat(settings.MEDIA_ROOT + "/cache" + "/003_ant_image_0003.jpg") # a NParray object
image_list = Image.objects.annotate(score=np.dot(
JSONVectConverter.json_to_vect(F('signature')), query_signature.T
).astype(float)).order_by('score') #JSONVectConverter is a class of mine
return render(request, 'images/sorted.html', {'image_sorted': image_list})
of course it doesn't work. I think "F()" operator is out of scope...
If you're wondering, I'm writing an image retrieval webapp for my university thesis.
Thank you.
EDIT:
I found this that is quite the same problem (He use postgres instead of MySQL)
EDIT2: I just remember now what is the last solution I've adopted! First I pull out every vector from the DB and mantain it in RAM, then I make some simple computes to find the K-Nearest Neighbors. Then, I retrieve from the DB the respective image using its index (primary key). So I decouple this task from Django ORM. Here's the code (from the Rest API)
def query_over_db(query_signature, page):
query_signature = np.array(query_signature)
t0 = time.time()
descriptor_matrix = cache.get('descriptor_matrix')
id_vector = cache.get('id_vector')
if not descriptor_matrix:
id_vector = []
descriptor_matrix = []
images_dict = Image.objects.all().values('id', 'signature')
for image in images_dict:
s = image['signature']
descriptor = np.array(s)
descriptor_matrix.append(descriptor)
id_vector.append(image['id'])
cache.set('id_vector', id_vector)
cache.set('descriptor_matrix', descriptor_matrix)
t1 = time.time()
print("time to pull out the descriptors : " + str(t1 - t0))
t1 = time.time()
#result = np.abs(np.dot(descriptor_matrix, query_signature.T))
#result = np.sum((descriptor_matrix - query_signature)**2, axis=1)
result = ne.evaluate('sum((descriptor_matrix - query_signature)**2, axis=1)')
t2 = time.time()
print("time to calculate similarity: " + str(t2 - t1))
perm = np.argsort(result)[(page - 1) * 30:page * 30]
print(perm.shape)
print(len(id_vector))
perm_id = np.array(id_vector)[perm]
print(len(perm_id))
print("printing sort")
print(np.sort(result)[0])
t4 = time.time()
print("time to order the result: " + str(t4 - t2))
qs = Image.objects.defer('signature').filter(id__in=perm_id.tolist())
qs_new = []
for i in range(len(perm_id)):
qs_new.append(qs.get(id=perm_id[i]))
t3 = time.time()
print("time to get the results from the DB : " + str(t3 - t2))
print("total time : " + str(t3 - t0))
print(result[perm])
return qs_new
I didn't come close to trying something this complex, however I've solved a similar issue here:
Combining Django F, Value and a dict to annotate a queryset
I haven't tried this but you could give it a go:
from django.db.models import Case, When, FloatField
query_signature = extract_feat(settings.MEDIA_ROOT + "/cache" + "/003_ant_image_0003.jpg") # a NParray object
value_dict = {}
for image in Image.objects.all():
value_dict[image.signature] = np.dot(
JSONVectConverter.json_to_vect(image.signature),
query_signature.T
).astype(float)
whens = [
When(signature=k, then=v) for k, v in value_dict.items()
]
qs = Image.objects.all().annotate(
score=Case(
*whens,
default=0,
output_field=FloatField()
)
).order_by('score')
Hope it helps
So that's the final working code:
def image_sorted(request):
query_signature = extract_feat(settings.MEDIA_ROOT + "/cache" + "/001_accordion_image_0001.jpg") # a NParray object
#query_signature = extract_feat(settings.MEDIA_ROOT + "/cache" + "/003_ant_image_0003.jpg") # a NParray object
value_dict = {}
for image in Image.objects.all():
S = image.signature
value_dict[image.signature] = np.dot(
JSONVectConverter.json_to_vect(S),
query_signature.T
).astype(float)
whens = [
When(signature=k, then=v) for k, v in value_dict.items()
]
qs = Image.objects.all().annotate(
score=Case(
*whens,
default=0,
output_field=FloatField()
)
).order_by('-score')
for image in qs:
print(image.score)
return render(request, 'images/sorted.html', {'image_sorted': qs})
Thanks to Omar for helping me! Of course I'm still here if there are finer solutions.
Related
I have this function that generate the initial population for a genetic algorithm.
import random
def initial_pop(pop_size):
pop = list()
for i in range(pop_size):
aux = list()
for j in range(2):
signal = bin(random.randint(0, 1))[2:].zfill(1)
int_part = bin(random.randint(0, 2))[2:].zfill(2)
real_part = bin(random.randint(0, 5000))[2:].zfill(13)
x = ''.join([signal, int_part, real_part])
aux.append(x)
pop.append(aux)
return pop
population = initial_pop(5)
print(population)
The function returns something like this:
[['1001000001111101', '1100111001001010'], ['0000111110100111', '0011001000100011'], ['1000010101001101', '0000101001100011'], ['1100000011011010', '0011000010001110'], ['0100010101001010', '1001000010001110']]
And I have this function that make the crossover between parents:
def crossover(pai1, pai2, cross_rate):
p1x = pai1[0]
p1y = pai1[1]
p2x = pai2[0]
p2y = pai2[1]
if np.random.rand() < tx_cruzamento:
f1x = p1x[:PONTO_CRUZAMENTO_1] + p2x[PONTO_CRUZAMENTO_1:PONTO_CRUZAMENTO_2] + p1x[:PONTO_CRUZAMENTO_2]
f1y = p1y[:PONTO_CRUZAMENTO_1] + p2y[PONTO_CRUZAMENTO_1:PONTO_CRUZAMENTO_2] + p1y[:PONTO_CRUZAMENTO_2]
f2x = p2x[:PONTO_CRUZAMENTO_1] + p1x[PONTO_CRUZAMENTO_1:PONTO_CRUZAMENTO_2] + p2x[:PONTO_CRUZAMENTO_2]
f2y = p2y[:PONTO_CRUZAMENTO_1] + p1y[PONTO_CRUZAMENTO_1:PONTO_CRUZAMENTO_2] + p2y[:PONTO_CRUZAMENTO_2]
f1 = [f1x] + [f1y]
f2 = [f2x] + [f2y]
return [f1] + [f2]
else:
return [pai1] + [pai2]
The call for this function is:
children.append(crossover(selected[j], selected[j+1], CROSS_RATE))
However, this function return something like this:
[[['1100101011000010', '1101100011010000'], ['1010001010101010', '1010000010101000']], [['1010101011010010', '1010100011001100'], ['1100010000100001', '1011000111101000']], [['1100010011000010', '1101000111010000'], ['1100001011000100', '1011000010110001']]]
How do I format the output to look exactly like the first one? The one that generated the initial population? I tried everything ([f1] + [f2], [[f1]+[f2]], [f1, f2], etc)
The problem seems to be in how selected is defined. When I call your cross-over function like this:
crossover(population[0],population[1],0.3)
I get output which looks like:
[['0000110000101001', '1010100101111111'], ['0010101110111110', '0001001010110001'], ['0100100110000010', '0100111100011001'], ['0011000100010010', '0010110110111111'], ['0010010111000110', '0010010101111011']]
[['0000110000101001', '1010100101111111'], ['0010101110111110', '0001001010110001']]
which matches the form of your intended output.
This suggests that the problem lies with code that you didn't show. Alternatively, crossover needs to do some preprocessing to extract the data that it needs from that which is passed to it. If this isn't the case, you really need to provide the missing details that would allow others to replicate the problem.
I was making my automatic stock strategy yield calculation program with Python. Here's my code:
import FinanceDataReader as fdr
import numpy as np
# ...(more modules for python)
pd.options.display.float_format = '{:.5f}'.format
file_list = os.listdir('/home/sejahui/projects/stock_data_excel')
for i in range(20):
os.chdir('/home/sejahui/projects/stock_data_excel')
odd = file_list[i]
data = pd.read_excel('/home/sejahui/projects/stock_data_excel/'+str(odd))
def calMACD(data, short=5, long=25, signal=9):
data.sort_index()
data['MVA_25']=data['Close'].ewm(span=long, adjust=False).mean()
data['MVA_5']=data['Close'].ewm(span=short, adjust=False).mean()
data['MACD']=data['Close'].ewm(span=short, adjust=False).mean() - data['Close'].ewm(span=long, adjust=False).mean()
data['Signal']=data['MACD'].ewm(span=signal, adjust=False).mean( )
#data['Buy_sign']=(data['MACD']-data['Signal']) >=600
data['Buy_sign']=np.where(data['MACD']-data['Signal'] >=451, 'Buy' , 'Sell' )
#data['Target_1']=(data['Close']-data['Close'].shift(1))/data['Close'].shift(1)*100
#data['Target_1']=np.where(data['Buy_sign']=='Buy', (data['Change'])+1,1)
#data['Target_2']=np.where(data['Buy_sign']=='Sell', (data['Change'])+1,1)
#data['Real_world']= 1000000*data['Target_1']
#data['Real_world_2']= 1000000*data['Target_2']
#data['Condition'] = np.where(data['Real_world']<1000000, data['Real_world']-data['Real_world'].shift(-2),1)
##data['Condition_2'] = np.where(data['Real_world']<1000000, data['Target_1'].shift(-2),1)
#data['Moneyflow'] =
#plt.plot(data['Date'], data['Real_world'])
#data[data.Buy_sign !='Sell']
'''
data['Target_1']=np.where(data['Buy_sign']=='Buy', data['Change'],1)
data['Target_2']=np.where(data['Buy_sign']=='Sell', data ['Change'],1)
data['Yield']=np.where(data['Buy_sign']=='Sell', data['Target_1']/data['Target_2'],1 )
'''
'''
data['Result']=data['Target_1'].cumprod()
data['Result_2']=data['Target_2'].cumprod()
data['??????'] = data['Result'] - data['Result_2']
'''
return data
Adjusted = calMACD(data)
Adjusted.drop(['Change'], axis=1, inplace = True)
Filtered = Adjusted[Adjusted.Buy_sign!='Sell'].copy()
#print(Filtered)
#Filtered = (Adjusted.Buy_sign =='Buy') #(Adjusted.Condition = 1.0)
#Master = Adjusted.loc[Adjusted,['Date','Buy_sign','Target_1','Real_world',]]
#print(Adjusted)
def backtester(Filtered):
Filtered['Change'] = ((Filtered['Close'] - Filtered['Close'].shift(1)) / Filtered['Close'].shift(1))+1
#data['Target_1']=np.where(data['Buy_sign']=='Buy', (data['Change'])+1,1)
Filtered['Real_world'] = 1000000*Filtered['Change']
#Filtered['Condition'] = np.where(Filtered['Real_world']<1000000, Filtered['Real_world'].shift(-2)-Filtered['Real_world'],1)
Filtered['Condition'] = np.where(Filtered['Real_world']<1000000, Filtered['Change'].shift(-2),1)
#Filtered['Target_1'] = np.where(Filtered['Buy_sign']=='Buy', (Filtered['Change'])+1,1)
#Filtered['Condition'] = np.where(Filtered['Real_world']<1000000, Filtered['Real_world'].shift(-2)-Filtered['Real_world'],1)
return Filtered
s = backtester(Filtered)
e = s[s.Condition!=1.00000]
x = e.dropna()
y = x['Condition']
list_1 = []
write_wb = Workbook()
write_ws = write_wb.create_sheet('MACD&Signal gap data sheet')
write_ws = write_wb.active
write_ws['A1'] = 'Name'
write_ws['B1'] = 'Profit'
try:
print(geometric_mean(y)*1000000*12)
except StatisticsError as e:
print ('Sell is empty':',odd)
else:
d = (geometric_mean(y)*1000000*12)
print(d,odd)
list_1.insert(i,d)
Print(list_1)
Here's the part where I'm troubling with:
s = backtester(Filtered)
e = s[s.Condition!=1.00000]
x = e.dropna()
y = x['Condition']
list_1 = []
try:
print(geometric_mean(y)*1000000*12)
except StatisticsError as e:
print ('Sell is empty':',odd)
else:
d = (geometric_mean(y)*1000000*12)
print(d)
list_1.insert(d)
print(list_1)
When I initiate the code where I am having problems, list only saves the last result of 'try, except, else' function. My intention was saving all the results. What change should I give to save all the results?
Here's the output of the list:
[11772769.197974786]
Your problem is that you are using insert instead of append and the main difference that insert takes a second argument for the position that you want to insert your element at and when none is provided it is 0 by default so you are consistently inserting at the same index resulting in a list with only the last element at the first position.
To fix that simply use append instead.
else:
d = (geometric_mean(y)*1000000*12)
print(d)
list_1.append(d)
You want to use append, not insert. see Python Data Structures
Change list_1.insert(d) to list_1.append(d)
The insert is defaulting to index 0 and just updating it each time.
Edit: Just noticed your answer is in the question title.
I've been trying to set up a script to automatically assign connector displacement boundary conditions. When I run the script it all looks fine in the GUI (wires are created, BCs are created and assigned the right value), but when I submit I get the following error: "Element connectivity is missing for element x of type "CONN3D2" and the element connectivity is in fact missing in the input file. I assign the edges by using the midpoints between the wire start and ends, but for some reason it doesn't assign them to the elements. This is my connector assignment function:
def assignConnectors(self):
p = self.m.parts[self.partName]
a = self.m.rootAssembly
a.Instance(name=self.instanceName, part=p, dependent=ON)
e = a.edges
n = a.instances[self.instanceName].nodes
#allelements = p.Set(name='allElements', elements=self.listObjElem)
elset = a.instances[self.instanceName].elements
elsetAssembly = a.Set('assemblyElements', elements=elset)
a.regenerate()
v1 = a.instances[self.instanceName].vertices
rows = len(self.listConstraints)
columns = len(self.listConstraints[0])
total = rows*columns
listObjNode=[];
self.listObjElem=[];
self.listObjConnector=[];
for j,pairElem in enumerate(self.listElem):
p1 = a.getCoordinates(self.listNodes[pairElem[0]-1])
p2 = a.getCoordinates(self.listNodes[pairElem[1]-1])
#print(p1,p2)
wires = a.WirePolyLine(points=((p1,p2),), mergeType=IMPRINT, meshable=OFF)
a.regenerate()
pt1 = a.getCoordinates(self.listNodes[pairElem[0]-1])
pt2 = a.getCoordinates(self.listNodes[pairElem[1]-1])
print(pt1,pt2)
pt11 = np.asarray(pt1[0])
pt12 = np.asarray(pt1[1])
pt13 = np.asarray(pt1[2])
pt21 = np.asarray(pt2[0])
pt22 = np.asarray(pt2[1])
pt23 = np.asarray(pt2[2])
new_p1 = (pt11+pt21)/2
new_p2 = (pt12+pt22)/2
new_p3 = (pt13+pt23)/2
new_p = tuple([new_p1,new_p2,new_p3])
print(new_p)
a = self.m.rootAssembly
e = a.edges
edges1 = e.findAt((new_p, ))
print(edges1)
region = a.Set(edges = edges1, name='Set'+str(j))
self.m.ConnectorSection(name='ConnSect-1'+str(j),translationalType=AXIAL)
csa = a.SectionAssignment(sectionName='ConnSect-1'+str(j), region=region)
self.m.ConnDisplacementBC(name='BC-'+str(j+total), createStepName=self.stepName, fastenerSetName='Set'+str(j), u1=float(self.listElongations[j]), u2=UNSET, u3=UNSET, ur1=UNSET, ur2=UNSET, ur3=UNSET, amplitude=UNSET, fixed=OFF, distributionType=UNIFORM)
a.regenerate()
Am I assigning the elements wrong somehow?
Thanks a lot for any help!
I'm trying to build a PID controller on Python. Below is so far my implementation. Although the syntax is coherent, this is still pseudocode.
def PID(self, Kp, Ki, Kd, reference_velocity, vehicle_velocity, current_time, last_update_time):
desired_velocity = 0
Kp = 0
Ki = 0
Kd = 0
error = reference_velocity - vehicle_velocity
delta_time = current_time - last_update_time
proportional = Kp * error
integral = Ki * error
derivative = Kd * ((error - last_error) / (delta_time))
desired_velocity = proportional + integral + derivative
return desired_velocity
I am trying to figure out the last_error through what I already have given here but I couldn't figure it out.
Store the errors or the last error in a container in the function's outer scope.
errors = []
def PID(...):
...
errors.append(error)
return desired_velocity
Then in the function that last error can be retrieved with.
...
last_error = errors[-1]
Or you could make your controller a class whose instances are initialized with constants and starting parameters, and keeps records (a container like a list) of past errors. The calculation for the current/next output control signal would be a method that could be called with current parameter values.
Using your algorithm - something like this.
class PID:
def __init__(self,setpoint,t_now,Kp=0,Ki=0,Kd=0):
self.setpoint = setpoint
self.t_last = t_now
self.Kp = Kp
self.Ki = Ki
self.Kd = Kd
self.last_error = 0
def adjust(self,process,t_now):
error = process - self.setpoint
delta_time = t_now - self.t_last
proportional = self.Kp * error
integral = self.Ki * error
derivative = self.Kd * ((error - self.last_error) / (delta_time))
desired_process = proportional + integral + derivative
self.t_last = t_now
self.last_error = error
return desired_process
Usage:
v_controller = PID(setpoint=66, t_now=0, Kp=.2, Kd=.1, Ki=1)
# or
#parameters = {'setpoint':66, 't_now':0, 'Kp':.2, 'Kd':.1, 'Ki':1}
#v_controller = PID(**parameters)
print(v_controller.adjust(66.1,1))
print(v_controller.adjust(66.2,2))
print(v_controller.adjust(65.9,3))
print('-------------')
parameters = {'setpoint':66, 't_now':0, 'Kp':.2, 'Kd':.1, 'Ki':1.5}
v_controller = PID(**parameters)
print(v_controller.adjust(66.1,1))
print(v_controller.adjust(66.2,2))
print(v_controller.adjust(65.9,3))
print('-------------')
parameters = {'setpoint':66, 't_now':0, 'Kp':1.25, 'Kd':.2, 'Ki':.5}
v_controller = PID(**parameters)
print(v_controller.adjust(66.1,1))
print(v_controller.adjust(66.2,2))
print(v_controller.adjust(65.9,3))
Result
>>>
0.12999999999999262
0.2500000000000043
-0.1499999999999929
-------------
0.17999999999998975
0.3500000000000057
-0.19999999999999005
-------------
0.1949999999999889
0.37000000000000666
-0.2349999999999895
>>>
I have a google earth engine javascript code to detect water pixel in the closest date SAR imagery. Link of the code: https://code.earthengine.google.com/0a35eea49123a5390b822bac7afc1b0c. I can run the code in GEE and returning exactly what I required (1 if the location is above water and 0 when it above land).
I have tried to develop the following code which is returning the dictionary, not the single expected output.
import ee, datetime
ee.Initialize()
waterThresh = -16;
angle_threshold_1 = ee.Number(45.4);
angle_threshold_2 = ee.Number(31.66);
class AltimeterExtraction(object):
def __init__(self, locationfile = './Bin/Location_Data.txt'):
filecontent = open(locationfile, 'r')
content = csv.DictReader(filecontent, delimiter='\t')
def watertestpoint(self, lat=10.55587,lon=89.89789, date1='2019-04-05'):
lat = ee.Number(lat)
lon = ee.Number(lon)
datep = datetime.datetime.strptime(date1, "%Y-%m-%d")
date2 = datep + datetime.timedelta(days = -10)
point = ee.Geometry.Point([lon,lat])
S1 = ee.ImageCollection('COPERNICUS/S1_GRD').filterBounds(point).filterDate(date2, date1)
S1 = S1.map(self.maskByAngle)
S1 = S1.select('VV').median().rename('VV')
S1 = S1.focal_median(50,'circle','meters').rename('VV')
WaterMask = S1.lt(waterThresh)
flag = WaterMask.reduceRegion(**{
'reducer': ee.Reducer.mean(),
'geometry': point,
'scale': 10
});
return flag.get('VV')
def maskByAngle(self, img):
I = ee.Image(img)
angle = I.select('angle')
mask1 = angle.lt(angle_threshold_1)
mask2 = angle.gt(angle_threshold_2)
I = I.updateMask(mask1)
return I.updateMask(mask2)
P = AltimeterExtraction()
x= P.watertestpoint()
print x
Are there any ways to get the single value instead of the dictionary from python? I need the binary output (0 or 1) from the function.
You should add .getInfo() while printing to get the actual value from that point. Earth Engine process all of the data on the server side so you have to explicitly call .getInfo() to return the information.
Here is an example I used:
P = AltimeterExtraction()
x= P.watertestpoint(lat=20.5564,lon=94.818,date1='2019-03-30')
print(x.getInfo())