I have a google earth engine javascript code to detect water pixel in the closest date SAR imagery. Link of the code: https://code.earthengine.google.com/0a35eea49123a5390b822bac7afc1b0c. I can run the code in GEE and returning exactly what I required (1 if the location is above water and 0 when it above land).
I have tried to develop the following code which is returning the dictionary, not the single expected output.
import ee, datetime
ee.Initialize()
waterThresh = -16;
angle_threshold_1 = ee.Number(45.4);
angle_threshold_2 = ee.Number(31.66);
class AltimeterExtraction(object):
def __init__(self, locationfile = './Bin/Location_Data.txt'):
filecontent = open(locationfile, 'r')
content = csv.DictReader(filecontent, delimiter='\t')
def watertestpoint(self, lat=10.55587,lon=89.89789, date1='2019-04-05'):
lat = ee.Number(lat)
lon = ee.Number(lon)
datep = datetime.datetime.strptime(date1, "%Y-%m-%d")
date2 = datep + datetime.timedelta(days = -10)
point = ee.Geometry.Point([lon,lat])
S1 = ee.ImageCollection('COPERNICUS/S1_GRD').filterBounds(point).filterDate(date2, date1)
S1 = S1.map(self.maskByAngle)
S1 = S1.select('VV').median().rename('VV')
S1 = S1.focal_median(50,'circle','meters').rename('VV')
WaterMask = S1.lt(waterThresh)
flag = WaterMask.reduceRegion(**{
'reducer': ee.Reducer.mean(),
'geometry': point,
'scale': 10
});
return flag.get('VV')
def maskByAngle(self, img):
I = ee.Image(img)
angle = I.select('angle')
mask1 = angle.lt(angle_threshold_1)
mask2 = angle.gt(angle_threshold_2)
I = I.updateMask(mask1)
return I.updateMask(mask2)
P = AltimeterExtraction()
x= P.watertestpoint()
print x
Are there any ways to get the single value instead of the dictionary from python? I need the binary output (0 or 1) from the function.
You should add .getInfo() while printing to get the actual value from that point. Earth Engine process all of the data on the server side so you have to explicitly call .getInfo() to return the information.
Here is an example I used:
P = AltimeterExtraction()
x= P.watertestpoint(lat=20.5564,lon=94.818,date1='2019-03-30')
print(x.getInfo())
Related
I've been trying to set up a script to automatically assign connector displacement boundary conditions. When I run the script it all looks fine in the GUI (wires are created, BCs are created and assigned the right value), but when I submit I get the following error: "Element connectivity is missing for element x of type "CONN3D2" and the element connectivity is in fact missing in the input file. I assign the edges by using the midpoints between the wire start and ends, but for some reason it doesn't assign them to the elements. This is my connector assignment function:
def assignConnectors(self):
p = self.m.parts[self.partName]
a = self.m.rootAssembly
a.Instance(name=self.instanceName, part=p, dependent=ON)
e = a.edges
n = a.instances[self.instanceName].nodes
#allelements = p.Set(name='allElements', elements=self.listObjElem)
elset = a.instances[self.instanceName].elements
elsetAssembly = a.Set('assemblyElements', elements=elset)
a.regenerate()
v1 = a.instances[self.instanceName].vertices
rows = len(self.listConstraints)
columns = len(self.listConstraints[0])
total = rows*columns
listObjNode=[];
self.listObjElem=[];
self.listObjConnector=[];
for j,pairElem in enumerate(self.listElem):
p1 = a.getCoordinates(self.listNodes[pairElem[0]-1])
p2 = a.getCoordinates(self.listNodes[pairElem[1]-1])
#print(p1,p2)
wires = a.WirePolyLine(points=((p1,p2),), mergeType=IMPRINT, meshable=OFF)
a.regenerate()
pt1 = a.getCoordinates(self.listNodes[pairElem[0]-1])
pt2 = a.getCoordinates(self.listNodes[pairElem[1]-1])
print(pt1,pt2)
pt11 = np.asarray(pt1[0])
pt12 = np.asarray(pt1[1])
pt13 = np.asarray(pt1[2])
pt21 = np.asarray(pt2[0])
pt22 = np.asarray(pt2[1])
pt23 = np.asarray(pt2[2])
new_p1 = (pt11+pt21)/2
new_p2 = (pt12+pt22)/2
new_p3 = (pt13+pt23)/2
new_p = tuple([new_p1,new_p2,new_p3])
print(new_p)
a = self.m.rootAssembly
e = a.edges
edges1 = e.findAt((new_p, ))
print(edges1)
region = a.Set(edges = edges1, name='Set'+str(j))
self.m.ConnectorSection(name='ConnSect-1'+str(j),translationalType=AXIAL)
csa = a.SectionAssignment(sectionName='ConnSect-1'+str(j), region=region)
self.m.ConnDisplacementBC(name='BC-'+str(j+total), createStepName=self.stepName, fastenerSetName='Set'+str(j), u1=float(self.listElongations[j]), u2=UNSET, u3=UNSET, ur1=UNSET, ur2=UNSET, ur3=UNSET, amplitude=UNSET, fixed=OFF, distributionType=UNIFORM)
a.regenerate()
Am I assigning the elements wrong somehow?
Thanks a lot for any help!
Good morning, I have a problem when reading a large netCDF file in python, which contains meteorological information, that information must go through it to assemble the information and then insert it into the database, but the time it takes to go through and assemble the information is too much, I know there must be other ways to perform the same process more efficiently, currently I access the information through a for loop, below the code
content = nc.Dataset(pathFile+file)
XLONG, XLAT = content.variables["XLONG"], content.variables["XLAT"]
Times = content.variables["Times"] #Horas formar b 'b
RAINC = content.variables["RAINC"] #Lluvia
Q2 = content.variables["Q2"] #Humedad especifica
T2 = content.variables["T2"] #Temperatura
U10 = content.variables["U10"] #Viento zonal
V10 = content.variables["V10"] #Viento meridional
SWDOWN = content.variables["SWDOWN"] #Radiacion incidente
PSFC = content.variables["PSFC"] #Presion de la superficie
SST = content.variables["SST"] #Temperatura de la superficie del mar
CLDFRA = content.variables["CLDFRA"] #Fraccion de nubes
for c2 in range(len(XLONG[0])):
for c3 in range(len(XLONG[0][c2])):
position += 1
for hour in range(len(Times)):
dateH = getDatetimeInit(dateFormatFile.hour) if hour == 0 else getDatetimeForHour(hour, dateFormatFile.hour)
hourUTC = getHourUTC(hour)
RAINH = str(RAINC[hour][0][c2][c3])
Q2H = str(Q2[hour][0][c2][c3])
T2H = str(convertKelvinToCelsius(T2[hour][0][c2][c3]))
U10H = str(U10[hour][0][c2][c3])
V10H = str(V10[hour][0][c2][c3])
SWDOWNH = str(SWDOWN[hour][0][c2][c3])
PSFCH = str(PSFC[hour][0][c2][c3])
SSTH = str(SST[hour][0][c2][c3])
CLDFRAH = str(CLDFRA[hour][0][c2][c3] )
rowData = [idRun, functions.IDMODEL, idTime, position, dateH.year, dateH.month, dateH.day, dateH.hour, RAINH, Q2H, T2H, U10H, V10H, SWDOWNH, PSFCH, SSTH, CLDFRAH]
dataProcess.append(rowData)
I would use NumPy. Let us assume you have netCDF with 2 variables, "t2" and "slp". Then you could use the following code to vectorize your data:
#!//usr/bin/env ipython
# ---------------------
import numpy as np
from netCDF4 import Dataset
# ---------------------
filein = 'test.nc'
ncin = Dataset(filein);
tair = ncin.variables['t2'][:];
slp = ncin.variables['slp'][:];
ncin.close();
# -------------------------
tairseries = np.reshape(tair,(np.size(tair),1));
slpseries = np.reshape(slp,(np.size(slp),1));
# --------------------------
## if you want characters:
#tairseries = np.array([str(val) for val in tairseries]);
#slpseries = np.array([str(val) for val in slpseries]);
# --------------------------
rowdata = np.concatenate((tairseries,slpseries),axis=1);
# if you want characters, do this in the end:
row_asstrings = [[str(vv) for vv in val] for val in rowdata]
# ---------------------------
Nevertheless, I have a feeling that using strings is not very good idea. In my example, the conversion from numerical arrays to strings, took quite long time and therefore I did not implement it before concatenation.
If you want also some time/location information, you can do like this:
#!//usr/bin/env ipython
# ---------------------
import numpy as np
from netCDF4 import Dataset
# ---------------------
filein = 'test.nc'
ncin = Dataset(filein);
xin = ncin.variables['lon'][:]
yin = ncin.variables['lat'][:]
timein = ncin.variables['time'][:]
tair = ncin.variables['t2'][:];
slp = ncin.variables['slp'][:];
ncin.close();
# -------------------------
tairseries = np.reshape(tair,(np.size(tair),1));
slpseries = np.reshape(slp,(np.size(slp),1));
# --------------------------
## if you want characters:
#tairseries = np.array([str(val) for val in tairseries]);
#slpseries = np.array([str(val) for val in slpseries]);
# --------------------------
rowdata = np.concatenate((tairseries,slpseries),axis=1);
# if you want characters, do this in the end:
#row_asstrings = [[str(vv) for vv in val] for val in rowdata]
# ---------------------------
# =========================================================
nx = np.size(xin);ny = np.size(yin);ntime = np.size(timein);
xm,ym = np.meshgrid(xin,yin);
xmt = np.tile(xm,(ntime,1,1));ymt = np.tile(ym,(ntime,1,1))
timem = np.tile(timein[:,np.newaxis,np.newaxis],(1,ny,nx));
xvec = np.reshape(xmt,(np.size(tair),1));yvec = np.reshape(ymt,(np.size(tair),1));timevec = np.reshape(timem,(np.size(tair),1)); # to make sure that array's size match, I am using the size of one of the variables
rowdata = np.concatenate((xvec,yvec,timevec,tairseries,slpseries),axis=1);
In any case, with variable sizes (744,150,150), it took less than 2 seconds to vectorize 2 variables.
I'm working on a web scraping project, and have all the right code that returns me the json data in the format that I want if I used the #print command below, but when I got to run the same code except through Pandas Dataframe it only returns the first row of Data that I'm looking for. Just running the print, it returns the expected 17 rows of data I'm looking for. Dataframe to CSV gives me the first row only. Totally stumped! So grateful for anyone's help!
for item in response['body']:
DepartureDate = item['legs'][0][0]['departDate']
ReturnDate = item['legs'][1][0]['departDate']
Airline = item['legs'][0][0]['airline']['code']
Origin = item['legs'][0][0]['depart']
Destination = item['legs'][0][0]['destination']
OD = (Origin + Destination)
TrueBaseFare = item['breakdown']['baseFareAmount']
YQYR = item['breakdown']['fuelSurcharge']
TAX = item['breakdown']['totalTax']
TTL = item['breakdown']['totalFareAmount']
MARKEDUPTTL = item['breakdown']['totalCalculatedFareAmount']
MARKUP = ((MARKEDUPTTL - TTL) / (TTL)*100)
FBC = item['fareBasisCode']
#print(DepartureDate,ReturnDate,Airline,OD,TrueBaseFare,YQYR,TAX,TTL,MARKEDUPTTL,MARKUP,FBC)
MI = pd.DataFrame(
{'Dept': [DepartureDate],
'Ret': [ReturnDate],
'AirlineCode': [Airline],
'Routing': [OD],
'RealFare': [TrueBaseFare],
'Fuel': [YQYR],
'Taxes': [TAX],
'RealTotal': [TTL],
'AgencyTotal': [MARKEDUPTTL],
'Margin': [MARKUP],
'FareBasis': [FBC],
})
df = pd.DataFrame(MI)
pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns', None)
pd.set_option('display.width', None)
df.to_csv('MITest7.csv')
When you print all your values after the cycle, you will see that you get only the last values. To resolve this problem you need to create lists and put there your values.
Try this:
DepartureDate = []
ReturnDate = []
Airline = []
Origin = []
Destination = []
OD = []
TrueBaseFare = []
YQYR = []
TAX = []
TTL = []
MARKEDUPTTL = []
MARKUP = []
FBC = []
for item in response['body']:
DepartureDate.append(item['legs'][0][0]['departDate'])
ReturnDate.append(item['legs'][1][0]['departDate'])
Airline.append(item['legs'][0][0]['airline']['code'])
Origin.append(item['legs'][0][0]['depart'])
Destination.append(item['legs'][0][0]['destination'])
OD.append((Origin[-1] + Destination[-1]))
TrueBaseFare.append(item['breakdown']['baseFareAmount'])
YQYR.append(item['breakdown']['fuelSurcharge'])
TAX.append(item['breakdown']['totalTax'])
TTL.append(item['breakdown']['totalFareAmount'])
MARKEDUPTTL.append(item['breakdown']['totalCalculatedFareAmount'])
MARKUP.append(((MARKEDUPTTL[-1] - TTL[-1]) / (TTL[-1])*100))
FBC.append(item['fareBasisCode'])
I have a netcdf file with global wind data and I need to extract the component of wind UGRD& VGRD for my study area (lonmin=-2, lonmax=8, latmin=35 latmax=39). I need to have a text file with this format:
time series UGRD
VGRD
Example
19790101060000 (year month day hours)
3.28 5.26 (UGRD)
2.23 2.225 (VGRD)
I tried to do this operation with python. I succeeded to extract my study area in nc file but, I still tried to convert it to a text file but I've failed. Can someone help me to do it, please?
import numpy as np
import netCDF4
import netCDF4 as nc
import pandas as pd
import numpy as np
import csv
#### === User-inputs ====#####
one = nc.Dataset('1979.nc') ##load one of your nc datafiles
print one.variables ## Check variables names, say my variable names are lat, lon, pre
## Name of the variables
lat_name = 'latitude'
lon_name = 'longitude'
time_name = 'time'
data_name1 = 'UGRD_10maboveground'
data_name2 = 'VGRD_10maboveground'
## Select spatial range for which data to be extracted
mylat1 = 35
mylat2 = 39
mylon1 = -2
mylon2 = 8
##Give a name of your extracted datafile and define units
newfilename = 'Extracted_Data'
time_unit = 'day since 1979-01-01 00:00'
lat_unit = 'degrees_south'
lon_unit = 'degrees_east'
data_unit = 'm/s'
#### ======= Rest of the Code is Automated ========######
##Find pixel-range based on the provided lat-lon
lat = one.variables[lat_name][:]
lon = one.variables[lon_name][:]
ver_pix = []
for i in xrange(0, len(lat)):
if lat[i] >= mylat1 and lat[i] <= mylat2:
ver_pix.append(i)
y_min = min(ver_pix)
y_max = max(ver_pix)
print lat[min(ver_pix):max(ver_pix)]
hor_pix = []
for j in xrange(0,len(lon)):
if lon[j] >= mylon1 and lon[j] <= mylon2:
hor_pix.append(j)
x_min = min(hor_pix)
x_max = max(hor_pix)
print lon[min(hor_pix):max(hor_pix)]
check_range1 = one.variables[data_name1][:,y_min:y_max,x_min:x_max] ##pre:lat:lon =
time,y,x
check_range2 = one.variables[data_name2][:,y_min:y_max,x_min:x_max]
#print check_range
print check_range1.shape
print check_range2.shape
## Load all nc files in the directory from which data to be extracted
## ..for the selected area
f = nc.MFDataset('1979.nc')
alldata = f.variables[data_name1][:,y_min:y_max,x_min:x_max]
alldata = f.variables[data_name2][:,y_min:y_max,x_min:x_max]
lat1 = one.variables[lat_name][y_min:y_max]
lon1 = one.variables[lon_name][x_min:x_max]
#time = one.variables[time_name][:]
ncfile = nc.Dataset(''+str(newfilename)+'.nc','w')
ncfile.createDimension(time_name,len(alldata))
ncfile.createDimension(lat_name,len(lat1))
ncfile.createDimension(lon_name,len(lon1))
time = ncfile.createVariable(time_name,np.dtype('float32').char,(time_name,))
lats = ncfile.createVariable(lat_name,np.dtype('float32').char,(lat_name,))
lons = ncfile.createVariable(lon_name,np.dtype('float32').char,(lon_name,))
time.units = time_unit
lats.units = lat_unit
lons.units = lon_unit
time[:] = np.linspace(1,len(alldata),len(alldata))
lats[:] = lat1
lons[:] = lon1
newdata1 = ncfile.createVariable(data_name1,np.dtype('float32').char,
(time_name,lat_name,lon_name))
newdata2 = ncfile.createVariable(data_name2,np.dtype('float32').char,
(time_name,lat_name,lon_name))
newdata1.units = data_unit
newdata2.units = data_unit
newdata1[:] = alldata[:]
newdata2[:] = alldata[:]
dtime=netCDF4.num2date(time[:],time.units)
UGRD_ts=pd.Series(data_name2,index=dtime)
UGRD_ts.to_csv('data1.csv', index=True, header=True)
My first post here.
So I'm loading data into a variable called f1_data, then passing it to pm.removeDC() function to do some signal processing, and keeping the result into the same variable. But then, I want to replace only the column 8, with the original f1_data that I called raw_data and I can't figure it out why it doesn't work. Here are the functions. Help anyone?
inside file pm.py
def removeDC(data):
# define the filter
butter_order = 2
hp_cutoff_Hz = 1.0
b, a = signal.butter(butter_order, hp_cutoff_Hz/(fs_Hz / 2.0), 'highpass')
for i in range(1,9):
data[:,i] = signal.lfilter(b, a, data[:,i], 0)
return (data)
def get_epoch1(data, t_sec, epoch, f_tup, col):
#f_tup = (f_wdir, f_name, f_columns, out_save, out_dir, out_number, fig_width)
f_name = f_tup[1]
fig_width = f_tup[6]
epoch_boolvector = (t_sec >= epoch[0][0]) & (t_sec <= epoch[0][1])
epoch_timescale = t_sec[epoch_boolvector]
epoch_data = data[epoch_boolvector]
plt.figure(figsize=(fig_width,8), dpi=96)
plt.plot(epoch_timescale, epoch_data[:,col]);
plt.xlim(epoch_timescale[0], epoch_timescale[-1])
plt.show()
return (epoch_boolvector, epoch_timescale, epoch_data)
inside main file
#load the whole data
(f1_data, f1_data_indices, f1_timescale) = pm.load_data(f1_wdir, f1_name)
raw_data = f1_data[:] #create copy of f1_data
(f1ep1_boolvector, f1ep1_timescale, f1ep1_data) = pm.get_epoch1(f1_data, f1_timescale, f1_epochs[1], f1_tup, 8)
#--- filter data to remove DC (1Hz)
f1_data = pm.removeDC(f1_data)
# replace only channel 8 with original data
f1_data[:,8] = raw_data[:,8]
(f1ep2_boolvector, f1ep2_timescale, f1ep2_data) = pm.get_epoch1(f1_data, f1_timescale, f1_epochs[1], f1_tup, 8)
The solution is import copy and use copy.deepcopy function.
For further info check this link:
docs.python.org/2/library/copy.html
When I have raw_data = f1_data[:] I get, after pm.removeDC():
raw_data is f1_data: False
(raw_data == f1_data).all(): True
But when I have raw_data = copy.deepcopy(f1_data) I get, after pm.removeDC():
raw_data is f1_data: False
(raw_data == f1_data).all(): False