Develop Reference

How to use all the elements of the array using for loop?

actually, I need to put the returned values of the function (global_displacement(X)) into another running loop.
can someone please tell me how to obtain the required output?
and what idiotic mistake I have been doing.
as every time it gives me only the first([ 0, 0, X[0], X[1]]) OR
the last value([ X[20], X[21], X[53], X[54]]) in the output,
because of wrong indendation of "return j" in the below written code .
import numpy as np
X = [ 0.19515612, 0.36477665, 0.244737, 0.42873321, 0.16864666, 0.08636661, 0.05376605, -0.57201897, -0.00935055, -1.24923862, 0., -1.53111525, 0.00935055, -1.24923862, -0.05376605, -0.57201897, -0.1686466,
0.08636661, -0.244737, 0.42873321, -0.19515612, 0.36477665, 0.02279911, 0. , 0.3563355 , 0.01379104, 0. , 0.42289958, -0.00747999, 0. , 0.0825908, -0.02949519 , 0. , -0.57435396,
-0.04074819, 0. , -1.25069528 ,-0.02972642, 0. , -1.53227704, -0. , 0. , -1.25069528 , 0.02972642 , 0. , -0.57435396 , 0.04074819 , 0. , 0.0825908, 0.02949519, 0. ,
0.42289958, 0.00747999 , 0. , 0.3563355 , -0.01379104, -0.02279911]
def global_displacement(X):
global_displacements = np.array( [[ 0, 0, X[0], X[1]], [ X[0], X[1], X[2], X[3]], [ X[2], X[3],X[4], X[5]], [ X[4],X[5],X[6], X[7]],[ X[6],X[7],X[8],X[9]], [ X[8],X[9],X[10], X[11] ], [ X[10], X[11],X[12], X[13]], [ X[12], X[13],X[14], X[15]],[ X[14], X[15],X[16], X[17]],[ X[16], X[17],X[18], X[19]], [ X[18], X[19],X[20], X[21]],[ X[20], X[21], 0, 0],
[ X[0], X[1], X[23], X[24]], [ X[2], X[3], X[26],X[27]], [ X[4], X[5], X[29],X[30]], [ X[6], X[7], X[32],X[33]], [ X[8],X[9],X[35], X[36]], [ X[10], X[11], X[38], X[39]], [ X[12], X[13], X[41], X[42]] ,[ X[14], X[15], X[44], X[45]],[ X[16], X[17], X[47], X[48]],[ X[18], X[19], X[50], X[51]], [ X[20], X[21], X[53], X[54]] ] )
for i in (global_displacements):
j = i.reshape(4,1)
return j
print(global_displacement(X))
this is the expected output, and I need to put these values in another loop, by calling this function.
[[0. ]
[0. ]
[0.19515612]
[0.36477665]]
[[0.19515612]
[0.36477665]
[0.244737 ]
[0.42873321]]
[[0.244737 ]
[0.42873321]
[0.16864666]
[0.08636661]]
[[ 0.16864666]
[ 0.08636661]
[ 0.05376605]
[-0.57201897]]
[[ 0.05376605]
[-0.57201897]
[-0.00935055]
[-1.24923862]]
[[-0.00935055]
[-1.24923862]
[ 0. ]
[-1.53111525]]
[[ 0. ]
[-1.53111525]
[ 0.00935055]
[-1.24923862]]
[[ 0.00935055]
[-1.24923862]
[-0.05376605]
[-0.57201897]]
[[-0.05376605]
[-0.57201897]
[-0.1686466 ]
[ 0.08636661]]
[[-0.1686466 ]
[ 0.08636661]
[-0.244737 ]
[ 0.42873321]]
[[-0.244737 ]
[ 0.42873321]
[-0.19515612]
[ 0.36477665]]
[[-0.19515612]
[ 0.36477665]
[ 0. ]
[ 0. ]]
[[0.19515612]
[0.36477665]
[0. ]
[0.3563355 ]]
[[0.244737 ]
[0.42873321]
[0. ]
[0.42289958]]
[[0.16864666]
[0.08636661]
[0. ]
[0.0825908 ]]
[[ 0.05376605]
[-0.57201897]
[ 0. ]
[-0.57435396]]
[[-0.00935055]
[-1.24923862]
[ 0. ]
[-1.25069528]]
[[ 0. ]
[-1.53111525]
[ 0. ]
[-1.53227704]]
[[ 0.00935055]
[-1.24923862]
[ 0. ]
[-1.25069528]]
[[-0.05376605]
[-0.57201897]
[ 0. ]
[-0.57435396]]
[[-0.1686466 ]
[ 0.08636661]
[ 0. ]
[ 0.0825908 ]]
[[-0.244737 ]
[ 0.42873321]
[ 0. ]
[ 0.42289958]]
[[-0.19515612]
[ 0.36477665]
[ 0. ]
[ 0.3563355 ]]

Your function already converts everything into the right format except that the inner values should be stored into a list. For this you can use numpy.newaxis. It is used to add a new dimension to your array (good post about its functionality).
import numpy as np
def global_displacement(X):
global_displacements = np.array( [[ 0, 0, X[0], X[1]], [ X[0], X[1], X[2], X[3]], [ X[2], X[3],X[4], X[5]], [ X[4],X[5],X[6], X[7]],[ X[6],X[7],X[8],X[9]], [ X[8],X[9],X[10], X[11] ], [ X[10], X[11],X[12], X[13]], [ X[12], X[13],X[14], X[15]],[ X[14], X[15],X[16], X[17]],[ X[16], X[17],X[18], X[19]], [ X[18], X[19],X[20], X[21]],[ X[20], X[21], 0, 0],
[ X[0], X[1], X[23], X[24]], [ X[2], X[3], X[26],X[27]], [ X[4], X[5], X[29],X[30]], [ X[6], X[7], X[32],X[33]], [ X[8],X[9],X[35], X[36]], [ X[10], X[11], X[38], X[39]], [ X[12], X[13], X[41], X[42]] ,[ X[14], X[15], X[44], X[45]],[ X[16], X[17], X[47], X[48]],[ X[18], X[19], X[50], X[51]], [ X[20], X[21], X[53], X[54]] ] )
new_structure = global_displacements[:, :, np.newaxis]
return new_structure
X = [ 0.19515612, 0.36477665, 0.244737, 0.42873321, 0.16864666, 0.08636661, 0.05376605, -0.57201897, -0.00935055, -1.24923862, 0., -1.53111525, 0.00935055, -1.24923862, -0.05376605, -0.57201897, -0.1686466,
0.08636661, -0.244737, 0.42873321, -0.19515612, 0.36477665, 0.02279911, 0. , 0.3563355 , 0.01379104, 0. , 0.42289958, -0.00747999, 0. , 0.0825908, -0.02949519 , 0. , -0.57435396,
-0.04074819, 0. , -1.25069528 ,-0.02972642, 0. , -1.53227704, -0. , 0. , -1.25069528 , 0.02972642 , 0. , -0.57435396 , 0.04074819 , 0. , 0.0825908, 0.02949519, 0. ,
0.42289958, 0.00747999 , 0. , 0.3563355 , -0.01379104, -0.02279911]
result = global_displacement(X)
print(result)
Output:
[[[ 0. ]
[ 0. ]
[ 0.19515612]
[ 0.36477665]]
[[ 0.19515612]
[ 0.36477665]
[ 0.244737 ]
[ 0.42873321]]
[[ 0.244737 ]
[ 0.42873321]
[ 0.16864666]
[ 0.08636661]]
[[ 0.16864666]
[ 0.08636661]
[ 0.05376605]
[-0.57201897]]
[[ 0.05376605]
[-0.57201897]
[-0.00935055]
[-1.24923862]]
[[-0.00935055]
[-1.24923862]
[ 0. ]
[-1.53111525]]
[[ 0. ]
[-1.53111525]
[ 0.00935055]
[-1.24923862]]
[[ 0.00935055]
[-1.24923862]
[-0.05376605]
[-0.57201897]]
[[-0.05376605]
[-0.57201897]
[-0.1686466 ]
[ 0.08636661]]
[[-0.1686466 ]
[ 0.08636661]
[-0.244737 ]
[ 0.42873321]]
[[-0.244737 ]
[ 0.42873321]
[-0.19515612]
[ 0.36477665]]
[[-0.19515612]
[ 0.36477665]
[ 0. ]
[ 0. ]]
[[ 0.19515612]
[ 0.36477665]
[ 0. ]
[ 0.3563355 ]]
[[ 0.244737 ]
[ 0.42873321]
[ 0. ]
[ 0.42289958]]
[[ 0.16864666]
[ 0.08636661]
[ 0. ]
[ 0.0825908 ]]
[[ 0.05376605]
[-0.57201897]
[ 0. ]
[-0.57435396]]
[[-0.00935055]
[-1.24923862]
[ 0. ]
[-1.25069528]]
[[ 0. ]
[-1.53111525]
[ 0. ]
[-1.53227704]]
[[ 0.00935055]
[-1.24923862]
[ 0. ]
[-1.25069528]]
[[-0.05376605]
[-0.57201897]
[ 0. ]
[-0.57435396]]
[[-0.1686466 ]
[ 0.08636661]
[ 0. ]
[ 0.0825908 ]]
[[-0.244737 ]
[ 0.42873321]
[ 0. ]
[ 0.42289958]]
[[-0.19515612]
[ 0.36477665]
[ 0. ]
[ 0.3563355 ]]]

First off, you don't need .reshape to transform a 1D array of N elements into a 2D array that's N by 1. You can just add a dimension to the array.
Second, you generally don't want to write loops to handle a Numpy array. You want to use Numpy tools to process everything at once. Just think about the problem in the full number of dimensions: you want to transform a 2D array that's M by N, into a 3D one that's M by N by 1. That's... still just adding a dimension to the array.
So:
global_displacements = np.array(...)
return global_displacements[..., np.newaxis]

Single value decomposition(SVD) issue on 3D data (Python)

I have a trajectory which contains several frames of some 3D data, which looks like the following (I am posting the whole frame for the sake of reproducibilty of my problem):
data1= [[ 89.29, 57.35 , 107.81999 ],
[ 91.37 , 60.39 , 109.119995],
[ 89.130005 , 61.289997 , 112.12999 ],
[ 89.45 , 57.729996 ,113.509995],
[ 93.149994 , 58.199997 ,114.20999 ],
[ 92.03999 , 61.21 ,116.44 ],
[ 89.48999 , 58.71 ,117.979996],
[ 92.42 , 56.399998 ,118.84 ],
[ 94.25 , 59.14 ,120.73999 ],
[ 91.44 , 59.62 ,123.28999 ],
[ 91.25 , 55.739998 ,124. ],
[ 94.95 , 55.829998 ,124.67 ],
[ 94.36 , 58.699997 ,127.079994],
[ 91.5 , 57.05 ,129.05 ],
[ 93.16 , 53.57 ,129.28 ],
[ 96.36 , 55.35 ,130.5 ],
[ 94.15 , 57.53 ,133. ],
[ 92.24 , 54.42 ,134.18 ],
[ 95.479996 , 52.35 ,134.88 ],
[ 96.81 , 55.429996 ,136.77 ],
[ 93.509995 , 55.73 ,138.76 ],
[ 94.06999 , 51.98 ,139.5 ],
[ 97.63 , 52.929996 ,140.72 ],
[ 96.08 , 55.72 ,142.92 ],
[ 93.63999 , 53.269997 ,144.65 ],
[ 96.149994 , 50.45 ,144.79999 ],
[ 99.10999 , 52.1 ,146.4 ],
[ 97.369995 , 54.16 ,149. ],
[ 94.2 , 55.65 ,150.56 ]]
I want to fit the line to my data. After checking answers of previous question (1), I borrowed/wrote the following code :
def Range(data):
x=[]
y=[]
for i in range(0, len(data.T)):
x.append(np.mean(data.T[i])- np.std(data.T[i]))
y.append(np.mean(data.T[i]) + np.std(data.T[i]))
normx,normy=np.linalg.norm(x),np.linalg.norm(y)
minimum=normx-(normx+normy)/2
maximum=normy-(normx+normy)/2
return(minimum,maximum)
def axis(data):
minimum,maximum=Range(data)
datamean = data.mean(axis=0)
uu, dd, vv = np.linalg.svd(data - datamean)
linepts = vv[0] * np.mgrid[minimum:maximum:2j][:, np.newaxis]
linepts += datamean
return(linepts)
The value of vv[0] turns out to be [ 0.15970461,-0.17264067,0.9719515 ], which looks perfect for my further usage. However, for following the data :
data2= [ 90.993996 , 62.075 ,108.487 ],
[ 88.036995 , 59.8525 ,109.3855 ],
[ 90.5945 , 57.614998 ,111.061005],
[ 92.17 , 60.6205 ,112.681496],
[ 88.934006 , 61.622 ,114.4255 ],
[ 88.077995 , 57.929 ,115.34 ],
[ 91.642 , 57.3 ,116.81049 ],
[ 91.431496 , 60.4655 ,118.813 ],
[ 88.269 , 59.22 ,120.685 ],
[ 89.883995 , 55.7975 ,121.2585 ],
[ 93.115 , 57.497 ,122.68849 ],
[ 91.090004 , 59.724 ,125.11 ],
[ 89.355 , 56.712498 ,126.7305 ],
[ 92.6985 , 54.758 ,126.52 ],
[ 94.4685 , 57.383 ,128.4515 ],
[ 91.9065 , 57.297997 ,131.2145 ],
[ 91.99349 , 53.578995 ,131.1695 ],
[ 95.752495 , 53.376 , 132.0325 ],
[ 95.24799 , 55.990997, 134.80699 ],
[ 92.29199 , 54.0455 , 136.277 ],
[ 94.5055 , 50.9205 , 136.68399 ],
[ 97.5085 , 52.947 , 137.85399 ],
[ 95.353 , 54.6695 , 140.651 ],
[ 94.194496 , 51.2645 , 141.4345 ],
[ 97.6015 , 49.722 , 141.7245 ],
[ 99.26149 , 52.813496 ,143.35449 ],
[ 96.79849 , 53.233498, 146.2645 ],
[ 96.237 , 49.554 , 146.97299 ]]
The value of vv[0] turns out to be [-0.18894662 , 0.24432637,-0.9511066 ], which is in opposite sign from my expected result. I am unsure why it would produce such a result. Could anyone help me sort out this issue?
1 Fitting a line in 3D

The first principal component is a direction. Think of it as a line of infinite length. It doesn't matter if you flip this line, the direction remains the same.
In other words: the algorithm does not consider the order of the points that the object has travelled to. So for the algorithm, the object flying backwards through the points is just as valid as flying forwards.

How do I write an accumulator to squash an array around a midpoint?

Say I've got an array of arrays as follows:
array([[ 7108.4, -600. ],
[ 7101.3, -500. ],
[ 7094.3, -400. ],
[ 7087.2, -300. ],
[ 7080.1, -200. ],
[ 7073.1, -100. ],
[ 7058.3, 100. ],
[ 7051.3, 200. ],
[ 7044.2, 300. ],
[ 7037.1, 400. ],
[ 7030.1, 500. ],
[ 7023. , 600. ]])
This is a list of prices and quantities, with the positive numbers being buys and the negative numbers being sells.
If the current market price is 7050, I want to 'squash' my prices, as follows:
array([[ 7108.4, -600. ],
[ 7101.3, -500. ],
[ 7094.3, -400. ],
[ 7087.2, -300. ],
[ 7080.1, -200. ],
[ 7073.1, -100. ],
[ 7050. , 300. ], # Price & quantity changed here
[ 7044.2, 300. ],
[ 7037.1, 400. ],
[ 7030.1, 500. ],
[ 7023. , 600. ]])
and and do the same in thing in the other direction, so if the market price were 7085:
array([[ 7108.4, -600. ],
[ 7101.3, -500. ],
[ 7094.3, -400. ],
[ 7087.2, -300. ],
[ 7085. , -300. ], # Price & quantity change here
[ 7058.3, 100. ],
[ 7051.3, 200. ],
[ 7044.2, 300. ],
[ 7037.1, 400. ],
[ 7030.1, 500. ],
[ 7023. , 600. ]])
What's the cleanest way to write such a thing? Ideally, I'd like to write the squashing and the quantity grouping as two separate steps.

This is the kind of operations that are made easier by pandas.
>>> import numpy as np
>>> import pandas as pd
>>>
>>> t = np.array([[ 7108.4, -600. ],
... [ 7101.3, -500. ],
... [ 7094.3, -400. ],
... [ 7087.2, -300. ],
... [ 7080.1, -200. ],
... [ 7073.1, -100. ],
... [ 7058.3, 100. ],
... [ 7051.3, 200. ],
... [ 7044.2, 300. ],
... [ 7037.1, 400. ],
... [ 7030.1, 500. ],
... [ 7023. , 600. ]])
>>>
>>> df = pd.DataFrame(t, columns=['price', 'quantity'])
>>> (df.price > 7050) & (df.quantity > 0)
0 False
1 False
2 False
3 False
4 False
5 False
6 True
7 True
8 False
9 False
10 False
11 False
dtype: bool
>>> df[(df.price > 7050) & (df.quantity > 0)]
price quantity
6 7058.3 100.0
7 7051.3 200.0
>>> df[(df.price > 7050) & (df.quantity > 0)].quantity.sum()
300.0
>>> df[(df.price < 7085) & (df.quantity < 0)]
price quantity
4 7080.1 -200.0
5 7073.1 -100.0
>>> df[(df.price < 7085) & (df.quantity < 0)].quantity.sum()
-300.0
You could still stick with numpy if you want:
>>> import numpy as np
>>> import pandas as pd
>>>
>>> t = np.array([[ 7108.4, -600. ],
... [ 7101.3, -500. ],
... [ 7094.3, -400. ],
... [ 7087.2, -300. ],
... [ 7080.1, -200. ],
... [ 7073.1, -100. ],
... [ 7058.3, 100. ],
... [ 7051.3, 200. ],
... [ 7044.2, 300. ],
... [ 7037.1, 400. ],
... [ 7030.1, 500. ],
... [ 7023. , 600. ]])
>>>
>>>
>>> t[:,0]
array([ 7108.4, 7101.3, 7094.3, 7087.2, 7080.1, 7073.1, 7058.3,
7051.3, 7044.2, 7037.1, 7030.1, 7023. ])
>>> t[:,0] > 7050
array([ True, True, True, True, True, True, True, True, False,
False, False, False], dtype=bool)
>>> (t[:,0] > 7050) & (t[:,1] > 0)
array([False, False, False, False, False, False, True, True, False,
False, False, False], dtype=bool)
>>> t[(t[:,0] > 7050) & (t[:,1] > 0)]
array([[ 7058.3, 100. ],
[ 7051.3, 200. ]])
>>> t[(t[:,0] < 7085) & (t[:,1] < 0)]
array([[ 7080.1, -200. ],
[ 7073.1, -100. ]])
>>> t[(t[:,0] < 7085) & (t[:,1] < 0)][:,1]
array([-200., -100.])
>>> t[(t[:,0] < 7085) & (t[:,1] < 0)][:,1].sum()
-300.0

since Eric answer is great but as per your comment :
Is possible to modify the original array, so that the output is the
full list of orders with changes inside?
Here is my approach without numpy :
array=[[ 7108.4, -600. ],
[ 7101.3, -500. ],
[ 7094.3, -400. ],
[ 7087.2, -300. ],
[ 7080.1, -200. ],
[ 7073.1, -100. ],
[ 7058.3, 100. ],
[ 7051.3, 200. ],
[ 7044.2, 300. ],
[ 7037.1, 400. ],
[ 7030.1, 500. ],
[ 7023. , 600. ]]
price=0
positive_value=7050
updated_list=[]
track=[]
for index,item in enumerate(array):
if item[0]>positive_value:
if item[1]>0:
price+=item[1]
track.append(index)
else:
updated_list.append(item)
else:
updated_list.append(item)
updated_list.insert(track[0],[positive_value,price])
print(updated_list)
output:
[[7108.4, -600.0], [7101.3, -500.0], [7094.3, -400.0], [7087.2, -300.0], [7080.1, -200.0], [7073.1, -100.0], [7050, 300.0], [7044.2, 300.0], [7037.1, 400.0], [7030.1, 500.0], [7023.0, 600.0]]
for second one:
array=[[ 7108.4, -600. ],
[ 7101.3, -500. ],
[ 7094.3, -400. ],
[ 7087.2, -300. ],
[ 7080.1, -200. ],
[ 7073.1, -100. ],
[ 7058.3, 100. ],
[ 7051.3, 200. ],
[ 7044.2, 300. ],
[ 7037.1, 400. ],
[ 7030.1, 500. ],
[ 7023. , 600. ]]
price=0
positive_value=7085
updated_list=[]
track=[]
for index,item in enumerate(array):
if item[0] < positive_value:
if item[1] < 0:
price+=item[1]
track.append(index)
else:
updated_list.append(item)
else:
updated_list.append(item)
updated_list.insert(track[0],[positive_value,price])
print(updated_list)
output:
[[7108.4, -600.0], [7101.3, -500.0], [7094.3, -400.0], [7087.2, -300.0], [7085, -300.0], [7058.3, 100.0], [7051.3, 200.0], [7044.2, 300.0], [7037.1, 400.0], [7030.1, 500.0], [7023.0, 600.0]]

How to get random forest regression performance output in Python like that produced in R?

In R, I can easily get the performance of a random forest like the following.
How can I get the similar stuff in Python easily? Thanks a lot.
Summary of the Random Forest Model
==================================
Number of observations used to build the model: 35
Missing value imputation is active.
Call:
randomForest(formula = rank ~ .,
data = crs$dataset[crs$sample, c(crs$input, crs$target)],
ntree = 500, mtry = 3, importance = TRUE, replace = FALSE, na.action = na.roughfix)
Type of random forest: regression
Number of trees: 500
No. of variables tried at each split: 3
Mean of squared residuals: 5.578147
% Var explained: 97.22
Variable Importance

Here there is a simple example using sklearn random forest . http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html#sklearn.ensemble.RandomForestClassifier
You can easily get the values that you are looking for.
In this example we are using an input matrix X with 2 variables and a binary output y.
from sklearn.ensemble import RandomForestClassifier
fr = RandomForestClassifier(n_estimators=100, oob_score=True).fit(X, y)
fr.n_estimators
Out[10]: 100
fr.oob_decision_function_
Out[11]:
array([[ 0.14285714, 0.85714286],
[ 0.86666667, 0.13333333],
[ 0.02631579, 0.97368421],
[ 1. , 0. ],
[ 0.97826087, 0.02173913],
[ 0.97826087, 0.02173913],
[ 0.20512821, 0.79487179],
[ 0.97368421, 0.02631579],
[ 0.77777778, 0.22222222],
[ 0. , 1. ],
[ 0. , 1. ],
[ 1. , 0. ],
[ 0.52380952, 0.47619048],
[ 0.43243243, 0.56756757],
[ 1. , 0. ],
[ 0. , 1. ],
[ 0.05714286, 0.94285714],
[ 0. , 1. ],
[ 1. , 0. ],
[ 0.76470588, 0.23529412],
[ 1. , 0. ],
[ 0. , 1. ],
[ 0.95454545, 0.04545455],
[ 0.9 , 0.1 ],
[ 0.02222222, 0.97777778],
[ 0.875 , 0.125 ],
[ 0.02857143, 0.97142857],
[ 1. , 0. ],
[ 0.58823529, 0.41176471],
[ 0. , 1. ],
[ 0.20512821, 0.79487179],
[ 0.97435897, 0.02564103],
[ 0.91176471, 0.08823529],
[ 0. , 1. ],
[ 0.30232558, 0.69767442],
[ 1. , 0. ],
[ 0.94444444, 0.05555556],
[ 0. , 1. ],
[ 0.075 , 0.925 ],
[ 0.05263158, 0.94736842],
[ 1. , 0. ],
[ 0. , 1. ],
[ 0.02702703, 0.97297297],
[ 0.91176471, 0.08823529],
[ 0.43243243, 0.56756757],
[ 0.08333333, 0.91666667],
[ 0.10526316, 0.89473684],
[ 0.93548387, 0.06451613],
[ 0.02857143, 0.97142857],
[ 0.53658537, 0.46341463],
[ 0.5 , 0.5 ],
[ 0.66666667, 0.33333333],
[ 1. , 0. ],
[ 0.55555556, 0.44444444],
[ 0.96666667, 0.03333333],
[ 0.97142857, 0.02857143],
[ 0. , 1. ],
[ 0. , 1. ],
[ 1. , 0. ],
[ 0.05882353, 0.94117647],
[ 0.94594595, 0.05405405],
[ 0.11904762, 0.88095238],
[ 0.92307692, 0.07692308],
[ 0.69767442, 0.30232558],
[ 1. , 0. ],
[ 0.12121212, 0.87878788],
[ 1. , 0. ],
[ 0.97727273, 0.02272727],
[ 1. , 0. ],
[ 0.87878788, 0.12121212],
[ 0.02380952, 0.97619048],
[ 0. , 1. ],
[ 0. , 1. ],
[ 0. , 1. ],
[ 0.10810811, 0.89189189],
[ 1. , 0. ],
[ 1. , 0. ],
[ 0.97619048, 0.02380952],
[ 0.54545455, 0.45454545],
[ 0.02380952, 0.97619048],
[ 0.07317073, 0.92682927],
[ 0.94285714, 0.05714286],
[ 0.25714286, 0.74285714],
[ 0. , 1. ],
[ 0. , 1. ],
[ 0.97560976, 0.02439024],
[ 0.11111111, 0.88888889],
[ 1. , 0. ],
[ 1. , 0. ],
[ 0.02857143, 0.97142857],
[ 0.97916667, 0.02083333],
[ 0. , 1. ],
[ 0.02564103, 0.97435897],
[ 0. , 1. ],
[ 0.32258065, 0.67741935],
[ 0.56410256, 0.43589744],
[ 1. , 0. ],
[ 0.92682927, 0.07317073],
[ 1. , 0. ],
[ 0.08823529, 0.91176471]])
fr.oob_score_
Out[12]: 0.87
fr.feature_importances_
Out[13]: array([ 0.82407373, 0.17592627])

List of all arrays not contained in other lists of arrays [closed]

Closed. This question does not meet Stack Overflow guidelines. It is not currently accepting answers.
This question does not appear to be about programming within the scope defined in the help center.
Closed 8 years ago.
Improve this question
I have a list of two dimensional points, represented as two element long lists/arrays. E.g.:
points =
[[ 10. , 10. ],
[ 11. , 10. ],
[ 10.5 , 9.1339746],
[ 10.5 , 10. ],
[ 10.75 , 9.5669873],
[ 10.25 , 9.5669873],
[ 2. , 2. ],
[ 3. , 2. ],
[ 2.5 , 1.1339746],
[ 2.5 , 2. ],
[ 2.75 , 1.5669873],
[ 2.25 , 1.5669873]]
I now want to have a list which does not contain certain elements of this first list.
exclude = [[2., 2.], [3., 2.], [2.5, 2.]]
Unfortunately
new_list = [p for p in points if p not in exclude]
will produce
[[ 10. , 10. ],
[ 11. , 10. ],
[ 10.5 , 9.1339746],
[ 10.5 , 10. ],
[ 10.75 , 9.5669873],
[ 10.25 , 9.5669873],
[ 2.75 , 1.5669873],
[ 2.25 , 1.5669873]]
instead of
[[ 10. , 10. ],
[ 11. , 10. ],
[ 10.5 , 9.1339746],
[ 10.5 , 10. ],
[ 10.75 , 9.5669873],
[ 10.25 , 9.5669873],
[ 2.5 , 1.1339746],
[ 2.75 , 1.5669873],
[ 2.25 , 1.5669873]]
It seems Python removes all elements here that have at least one element in common (and not all in common :/ ).
Is there any nice/short/elegant way to exclude elements if they are not completely contained in the first list?

Note: Since this question has been tagged numpy, I'm assuming points is a NumPy array. If that's true, you could generate a boolean mask (array) using np.logical_and and np.logical_or:
import numpy as np
points = np.array(
[[ 10. , 10. ],
[ 11. , 10. ],
[ 10.5 , 9.1339746],
[ 10.5 , 10. ],
[ 10.75 , 9.5669873],
[ 10.25 , 9.5669873],
[ 2. , 2. ],
[ 3. , 2. ],
[ 2.5 , 1.1339746],
[ 2.5 , 2. ],
[ 2.75 , 1.5669873],
[ 2.25 , 1.5669873]])
exclude = [[2., 2.], [3., 2.], [2.5, 2.]]
mask = np.logical_or.reduce(
[np.logical_and.reduce(
[points[:,idx] == ex[idx] for idx in range(len(ex))]) for ex in exclude])
new_points = points[~mask]
print(new_points)
prints
[[ 10. 10. ]
[ 11. 10. ]
[ 10.5 9.1339746]
[ 10.5 10. ]
[ 10.75 9.5669873]
[ 10.25 9.5669873]
[ 2.5 1.1339746]
[ 2.75 1.5669873]
[ 2.25 1.5669873]]

You can also view the 2D array as a 1D array and then use np.in1d.
#Using #unutbu array's.
def view_1d(arr):
return arr.view(np.dtype((np.void,arr.dtype.itemsize * arr.shape[1])))
points_1d=view_1d(points)
exclude_1d=view_1d(exclude)
print points[~np.in1d(points_1d,exclude_1d)]
[[ 10. 10. ]
[ 11. 10. ]
[ 10.5 9.1339746]
[ 10.5 10. ]
[ 10.75 9.5669873]
[ 10.25 9.5669873]
[ 2.5 1.1339746]
[ 2.75 1.5669873]
[ 2.25 1.5669873]]
Just to double check the trickery is working and some ballpark timings:
points=np.random.rand(1E6,2)
points=np.around(points,1)
exclude=np.random.rand(1E2,2)
exclude=np.around(exclude,1)
t = time.time()
mask1 = ~(np.in1d(view_1d(points),view_1d(exclude)))
print time.time()-t
#0.469238996506
t = time.time()
mask2 = ~np.logical_or.reduce(
[np.logical_and.reduce(
[points[:,idx] == ex[idx] for idx in range(len(ex))]) for ex in exclude])
print time.time()-t
#7.13628792763
#Just to check this is doing what I think its doing.
print np.all(mask1==mask2)
True
Timings are just for generating the mask. Both methods appear to scale similarly, I just showed large arrays to (hopefully) compensate for not using timeit.

Your code as posted is not syntactically valid.
It produces (once corrected) your desired result:
--
[[10.0, 10.0],
[11.0, 10.0],
[10.5, 9.1339746],
[10.5, 10.0],
[10.75, 9.5669873],
[10.25, 9.5669873],
[2.5, 1.1339746],
[2.75, 1.5669873],
[2.25, 1.5669873]]
See: http://ideone.com/7LOpa6
points = [[ 10. , 10. ],
[ 11. , 10. ],
[ 10.5 , 9.1339746],
[ 10.5 , 10. ],
[ 10.75 , 9.5669873],
[ 10.25 , 9.5669873],
[ 2. , 2. ],
[ 3. , 2. ],
[ 2.5 , 1.1339746],
[ 2.5 , 2. ],
[ 2.75 , 1.5669873],
[ 2.25 , 1.5669873]]
exclude = [[2., 2.], [3., 2.], [2.5, 2.]]
print [p for p in points if p not in exclude]