Related
I'm using code from the source given below to get the nearest "site".
Source: https://automating-gis-processes.github.io/site/notebooks/L3/nearest-neighbor-faster.html
My Code:
# Read data from a DB
test_df = pd.read_sql_query(sql, conn)
# Calculates distance between 2 points on a map using lat and long
# (Source: https://towardsdatascience.com/heres-how-to-calculate-distance-between-2-geolocations-in-python-93ecab5bbba4)
def haversine_distance(lat1, lon1, lat2, lon2):
r = 6371
phi1 = np.radians(float(lat1))
phi2 = np.radians(float(lat2))
delta_phi = np.radians(lat2 - lat1)
delta_lambda = np.radians(lon2- lon1)
a = np.sin(delta_phi / 2)**2 + np.cos(phi1) * np.cos(phi2) * np.sin(delta_lambda / 2)**2
res = r * (2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a)))
return np.round(res, 2)
test_df["actualDistance (km)"] = test_df.apply(lambda row: haversine_distance(row['ClientLat'],row['ClientLong'],row['actual_SLa'],row['actual_SLo']), axis=1)
test_gdf = geopandas.GeoDataFrame(test_df, geometry=geopandas.points_from_xy(test_df.ClientLong, test_df.ClientLat))
site_gdf = geopandas.GeoDataFrame(site_df, geometry=geopandas.points_from_xy(site_df.SiteLong, site_df.SiteLat))
#-------Set up the functions as shown in the tutorial-------
def get_nearest(src_points, candidates, k_neighbors=1):
"""Find nearest neighbors for all source points from a set of candidate points"""
# Create tree from the candidate points
tree = BallTree(candidates, leaf_size=15, metric='haversine')
# Find closest points and distances
distances, indices = tree.query(src_points, k=k_neighbors)
# Transpose to get distances and indices into arrays
distances = distances.transpose()
indices = indices.transpose()
# Get closest indices and distances (i.e. array at index 0)
# note: for the second closest points, you would take index 1, etc.
closest = indices[0]
closest_dist = distances[0]
# Return indices and distances
return (closest, closest_dist)
def nearest_neighbor(left_gdf, right_gdf, return_dist=False):
"""
For each point in left_gdf, find closest point in right GeoDataFrame and return them.
NOTICE: Assumes that the input Points are in WGS84 projection (lat/lon).
"""
left_geom_col = left_gdf.geometry.name
right_geom_col = right_gdf.geometry.name
# Ensure that index in right gdf is formed of sequential numbers
right = right_gdf.copy().reset_index(drop=True)
# Parse coordinates from points and insert them into a numpy array as RADIANS
left_radians = np.array(left_gdf[left_geom_col].apply(lambda geom: (geom.x * np.pi / 180, geom.y * np.pi / 180)).to_list())
right_radians = np.array(right[right_geom_col].apply(lambda geom: (geom.x * np.pi / 180, geom.y * np.pi / 180)).to_list())
# Find the nearest points
# -----------------------
# closest ==> index in right_gdf that corresponds to the closest point
# dist ==> distance between the nearest neighbors (in meters)
closest, dist = get_nearest(src_points=left_radians, candidates=right_radians)
# Return points from right GeoDataFrame that are closest to points in left GeoDataFrame
closest_points = right.loc[closest]
# Ensure that the index corresponds the one in left_gdf
closest_points = closest_points.reset_index(drop=True)
# Add distance if requested
if return_dist:
# Convert to meters from radians
earth_radius = 6371000 # meters
closest_points['distance'] = dist * earth_radius
return closest_points
closest_sites = nearest_neighbor(test_gdf, site_gdf, return_dist=True)
# Rename the geometry of closest sites gdf so that we can easily identify it
closest_sites = closest_sites.rename(columns={'geometry': 'closest_site_geom'})
# Merge the datasets by index (for this, it is good to use '.join()' -function)
test_gdf = test_gdf.join(closest_sites)
#Extracted closest site latitude and longitude for data analysis
test_gdf['CS_lo'] = test_gdf.closest_site_geom.apply(lambda p: p.x)
test_gdf['CS_la'] = test_gdf.closest_site_geom.apply(lambda p: p.y)
The code is a replica of the tutorial link I provided. And based on their explanation it should've worked.
To verify this data I got some statistical data using .describe(), and it showed me that the tutorials method did indeed give me a mean distance that was much closer than the distance in the actual data (792 m vs the actual distance which was 1.80 km).
Closest Distance generated using the BallTree method
Actual Distance in the data
However when I plotted them out on a map using plotly I noticed that the BallTree method's outputs weren't closer than the "actual" distance.
This is generally what the plotted data looks like (Blue: predetermined site, Red: site predicted using the BallTree method
Could someone help me track down the discrepancy
I'm not sure why this works but it did. I decided to just write the code based on the docs instead of following the tutorial and this worked:
# Build BallTree with haversine distance metric, which expects (lat, lon) in radians and returns distances in radians
dist = DistanceMetric.get_metric('haversine')
tree = BallTree(np.radians(site_df[['SiteLat', 'SiteLong']]), metric=dist)
test_coords = np.radians(test_df[['ClientLat', 'ClientLong']])
dists, ilocs = tree.query(test_coords)
The problem is that the tutorial code provides coordinates in Longitude, Latitude format instead of the Latitude, Longitude format BallTree anticipates. So you're measuring distances between inverted points.
If you swap the order of geom.x and geom.y in the coordinate parsing code you will get correct measurements.
# Parse coordinates from points and insert them into a numpy array as RADIANS
left_radians = np.array(left_gdf[left_geom_col].apply(lambda geom: (geom.y * np.pi / 180, geom.x * np.pi / 180)).to_list())
right_radians = np.array(right[right_geom_col].apply(lambda geom: (geom.y * np.pi / 180, geom.x * np.pi / 180)).to_list())
Hello i have a problem with calculating distances between cities from tsp library: http://www.math.uwaterloo.ca/tsp/world/countries.html. I have this set of data (cities in djibouti): http://www.math.uwaterloo.ca/tsp/world/dj38.tsp. I used this function to calculate distaces in this QaA here: http://comopt.ifi.uni-heidelberg.de/software/TSPLIB95/TSPFAQ.html. i programed this in python and now it looks like this, here is my code:
cityCoords = {
1:(11003.611100,42102.500000),
2:(11108.611100,42373.888900),
3:(11133.333300,42885.833300),
4:(11155.833300,42712.500000),
5:(11183.333300,42933.333300),
6:(11297.500000,42853.333300),
7:(11310.277800,42929.444400),
8:(11416.666700,42983.333300),
9:(11423.888900,43000.277800),
10:(11438.333300,42057.222200),
11:(11461.111100,43252.777800),
12:(11485.555600,43187.222200),
13:(11503.055600,42855.277800),
14:(11511.388900,42106.388900),
15:(11522.222200,42841.944400),
16:(11569.444400,43136.666700),
17:(11583.333300,43150.000000),
18:(11595.000000,43148.055600),
19:(11600.000000,43150.000000),
20:(11690.555600,42686.666700),
21:(11715.833300,41836.111100),
22:(11751.111100,42814.444400),
23:(11770.277800,42651.944400),
24:(11785.277800,42884.444400),
25:(11822.777800,42673.611100),
26:(11846.944400,42660.555600),
27:(11963.055600,43290.555600),
28:(11973.055600,43026.111100),
29:(12058.333300,42195.555600),
30:(12149.444400,42477.500000),
31:(12286.944400,43355.555600),
32:(12300.000000,42433.333300),
33:(12355.833300,43156.388900),
34:(12363.333300,43189.166700),
35:(12372.777800,42711.388900),
36:(12386.666700,43334.722200),
37:(12421.666700,42895.555600),
38:(12645.000000,42973.333300)
}
def calcCityDistances(coordDict):
cities = list(coordDict.keys())
n = len(cities)
distances = {}
latitude = []
longitude = []
RRR = 6378.388;
PI = 3.141592;
for i in range(1,n+1):
cityA = cities[i-1]
latA, longA = coordDict[cityA]
deg = int(latA)
Min = latA - deg
latitude.append(PI * (deg + 5 * Min / 3) / 180)
deg = int(longA);
Min = longA - deg;
longitude.append(PI * (deg + 5 * Min / 3) / 180)
for i in range(1,n+1):
for j in range(i + 1, n + 1):
q1 = cos(longitude[i-1] - longitude[j-1]);
q2 = cos(latitude[i-1] - latitude[j-1]);
q3 = cos(latitude[i-1] + latitude[j-1]);
key = frozenset((i, j))
distances[key] = {}
dist = RRR * acos(0.5 * ((1.0 + q1) * q2 - (1.0 - q1) * q3)) + 1.0
distances[key]['dist'] = dist
distances[key]['pher'] = init_fer
distances[key]['vis'] = 0
return distances
distances = calcCityDistances(cityCoords)
My problem is that the distances calculated in this algorithm are off mark in huge scale. average lenght of one route between cities is 10 000 km and the problem is that the optimal TSP route is 6635. you can imagine that when i apply this to my Ant Colony System algorithm the result is around 110 000 km. this is really different from 6 thousand. Can someone explain what am i doing wrong please ?
I'm not familiar with the distance calculation listed in the TSP FAQ. Here's the resource I've used in the past: http://www.movable-type.co.uk/scripts/latlong.html
He gives two great circle distance calculation methods. Neither one looks like the one TSP provided. But, they both produced a distance that seemed to match reality (that Diksa and Dikhil are about 31k apart).
The input data is in 1000ths of a degree, and I'm not sure if the conversion to radians given takes that into account.
Here's an implementation that might give you better results: note I updated the input data to degrees:
import cmath
import math
cityCoords = {
1:(11.0036111,42.1025),
2:(11.1086111,42.3738889)
}
def spherical_cosines(coordDict):
R = 6371; # kilometers
cities = list(coordDict.keys())
n = len(cities)
for i in range(1,n+1):
for j in range(i + 1, n + 1):
cityA = cities[i-1]
lat1, lon1 = coordDict[cityA]
cityB = cities[j-1]
lat2, lon2 = coordDict[cityB]
lat1_radians = math.radians(lat1)
lat2_radians = math.radians(lat2)
lon1_radians = math.radians(lon1)
lon2_radians = math.radians(lon2)
print('A={},{} B={},{}'.format(lat1_radians, lon1_radians, lat2_radians, lon2_radians))
delta_lon_radians = math.radians(lon2-lon1)
distance = cmath.acos(cmath.sin(lat1_radians) * cmath.sin(lat2_radians) + cmath.cos(lat1_radians) *
math.cos(lat2_radians) * cmath.cos(delta_lon_radians)) * R;
print('spherical_cosines distance={}'.format(distance))
spherical_cosines(cityCoords)
update:
The code you posted is not producing the correct distance values. Here's the first two cities using calcCityDistances and sperical cosines:
input loc=11003.6111, 42102.5
input loc=11108.6111, 42373.8889
radians A = 192.05631381917777,734.8329132074075
B=193.88890915251113,739.5740671363777
calcCityDistances distance = 8078.816781077703
input degrees A=11.0036111,42.1025 B=11.1086111,42.3738889
radians A=0.19204924330399503,0.7348272483209126
B=0.19388183901858905,0.7395638781792782
spherical_cosines> distance=(31.835225475974934+0j)
Units is kilometers. Spherical cosines produces approximately the right value. Is the code you're using the same as what you posted? Notice the radians conversion doesn't seem to take into account that the input is thousandths of a degree
I'm trying to write a program that will take a set of longitude & latitude coordinates from the user, convert them to x & y coordinates for a Mollweide projection map, and then report the value of the pixel at those coordinates (in this case, a noise temperature).
The map/data I'm using is the Haslam 408 MHz All Sky Survey which is provided as a Mollweide projection map. This data is in .fits format and is a large all-sky survey of noise in the 408 MHz band.
According to the Mollweide projection Wikipedia page, it is possible to use a Newton-Raphson iteration to convert from longitude/latitude to x/y map coordinates. I based the iteration scheme in my program largely on the methods from the Wikipedia page and in this GitHub post.
However, my program does not appear to be reporting the correct values for the longitude and latitude that I'm inputting. I largely suspect that one of two (or both) factors are contributing to this error:
The way that I'm implementing the iteration scheme is incorrect, and thus resulting in incorrect values being reported.
I don't properly understand what the radius value, R, represents in the iteration scheme. I can't find any literature on how to determine the proper R value beyond that "R is the radius of the globe to be projected." I assumed that this would be based upon the size of the map in pixels; in this case, the map image is 4096x2048 pixels, so I've tried using 2048, 1024, and simply 1 as the R values, to no avail.
Below I have provided my code for review:
from math import sin, cos, pi, sqrt, asin
from astropy.io import fits
hdulist = fits.open('data.fits')
hdulist.info()
data = hdulist[1].data
sqrt2 = sqrt(2)
def solveNR(lat, epsilon=1e-6): #this solves the Newton Raphson iteration
if abs(lat) == pi / 2:
return lat # avoid division by zero
theta = lat
while True:
nexttheta = theta - (
(2 * theta + sin(2 * theta) - pi * sin(lat)) /
(2 + 2 * cos(2 * theta))
)
if abs(theta - nexttheta) < epsilon:
break
theta = nexttheta
return nexttheta
def checktheta(theta, lat): #this function is also currently unused while debugging
return (2 * theta + sin(2 * theta), pi * sin(lat))
def mollweide(lat, lon, lon_0=0, R=1024):
lat = lat * pi / 180
lon = lon * pi / 180
lon_0 = lon_0 * pi / 180 # convert to radians
theta = solveNR(lat)
return (R * 2 * sqrt2 * (lon - lon_0) * cos(theta) / pi,
R * sqrt2 * sin(theta))
def inv_mollweide(x, y, lon_0=0, R=1024, degrees=True): # inverse procedure (x, y to lat, long). Currently unused
theta = asin(y / (R * sqrt2))
if degrees:
factor = 180 / pi
else:
factor = 1
return (
asin((2 * theta + sin(2 * theta)) / pi) * factor,
(lon_0 + pi * x / (2 * R * sqrt(2) * cos(theta))) * factor
)
def retrieve_temp(lat, long): #retrieves the noise temp from the data file after calling the mollweide function
lat = int(round(lat))
long = int(round(long))
coords = mollweide(lat, long)
x, y= coords
x = int(round(x))
y= int(round(y))
x = x-1
y = y-1
if x < 0:
x = x*(-1)
if y < 0:
y = y*(-1)
print("The noise temperature is: ",data[y, x],"K")
def prompt(): #this is the terminal UI
cont = 1
while cont == 1:
lat_cont = 1
while lat_cont == 1:
lat = float(input('Please enter the latitude: '))
lat_val = 1
while lat_val == 1:
if lat > 180 or lat < -180:
lat = float(input('Invalid input. Make sure your latitude value is in range -180 to 180 degrees \n'
'Please enter the latitude: '))
else:
lat_val = 0
lat_cont = 0
long_cont = 1
while long_cont == 1:
long = float(input('Please enter the longitude: '))
long_val = 1
while long_val == 1:
if long > 90 or long < -90:
long = float(input('Invalid input. Make sure your latitude value is in range -90 to 90 degrees \n'
'Please enter the latitude: '))
else:
long_val = 0
long_cont = 0
retrieve_temp(lat, long)
valid = 1
while valid == 1:
ans = input('Would you like to continue? Y or N: ').lower()
ans_val = 1
while ans_val ==1:
if not (ans == 'y' or ans == 'n'):
ans = input('Invalid input. Please answer Y or N to continue or exit: ')
elif ans == 'y':
ans_val = 0
cont = 1
valid = 0
elif ans == 'n':
ans_val = 0
cont = 0
valid = 0
prompt()
hdulist.close()
Apologies if I failed to follow typical Python conventions in the above code; I'm new to Python.
Your code looks reasonable. My advice about figuring out what's wrong:
(1) Try evaluating your mollweide and inv_mollweide functions at points for which you know what the results are supposed to be. E.g. points on the equator or prime meridian or something easy like that.
(2) Are your mollweide and inv_mollweide actually inverses? i.e. if you take your output from the one and put it into the other, you should get the original input back again.
(3) How do results change as you move around on the map? Do you get correct results in some areas (e.g. near middle of map) and not others? What happens as you get nearer to the edges? Does it gradually become more inaccurate or is there some threshold, beyond which you get grossly incorrect answers?
I think a feature of Newton's method is that it converges only if you're close enough to a solution to begin with, otherwise you can get anything. I don't know how close you have to be, for this problem.
This seems like a great problem. Good luck and have fun.
so this is going to sound really strange, but as far as I know there aren't actually any good Python libraries for Mollweide conversions, at least for Mollweide maps that span the entire earth. The EPSG system is bizarrely incomplete, which makes Pyproj more or less unusable for this purpose. I plan on using your code as part of a project of my own, but if you'd like to setup a github library and maybe even some kind of pip functionality (I'm fairly new to Python myself, so I'm really not who to ask on how to do that), I'm happy to give you credit.
I'm trying to calculate distance between two points, using latitude longitude and altitude (elevation).
I was using euklides formula in order to get my distance:
D=√((Long1-Long2)²+(Lat1-Lat2)²+(Alt1-Alt2)²)
My points are geographical coordinates and ofcourse altitude is my height above the sea.
I only have lat and lng, I'm using GOOGLE API Elevation to get my altitude.
I'm developing an application which calculates my traveled distance (on my skis). Every application which I have used, gets distance traveled with included altitude. Like #Endomondo or #Garmin I cannot get my distance in 2D space because true distances are going to vary from the ones I've returned.
Which formula would be the best to calculate my distance ? Ofcourse with included altitude.
I'm writing my app in Python, with PostGis.
You can calculate distance between flat coordinates in, say, meters by using geopy package or Vincenty's formula, pasting coordinates directly. Suppose the result is d meters. Then the total distance travelled is sqrt(d**2 + h**2) where h is the change in elevation in meters.
EDIT 2019: Since this answer, I composed a Q&A style example to answer similar questions (including this one as an example): How to calculate 3D distance (including altitude) between two points in GeoDjango.
In sort:
We need to calculate the 2D great-circle distance between 2 points using either the Haversine formula or the Vicenty formula and then we can combine it with the difference (delta) in altitude between the 2 points to calculate the Euclidean distance between them as follows:
dist = sqrt(great_circle((lat_1, lon_1), (lat_2, lon_2)).m**2, (alt_1 - alt_2)**2)
The solution assumes that the altitude is in meters and thus converts the great_circle's result into meters as well.
You can get the correct calculation by translating your coordinates from Polar (long, lat, alt) to Cartesian (x, y, z):
Let:
polar_point_1 = (long_1, lat_1, alt_1)
and polar_point_2 = (long_2, lat_2, alt_2)
Translate each point to it's Cartesian equivalent by utilizing this formula:
x = alt * cos(lat) * sin(long)
y = alt * sin(lat)
z = alt * cos(lat) * cos(long)
and you will have p_1 = (x_1, y_1, z_1) and p_2 = (x_2, y_2, z_2) points respectively.
Finally use the Euclidean formula:
dist = sqrt((x_2-x_1)**2 + (y_2-y_1)**2 + (z_2-z_1)**2)
I used the solution provided by John Moutafis but I didn't get a right answer.The formula needs some corrections. You will get the conversion of coordinates from Polar to Cartesian (x, y, z) at http://electron9.phys.utk.edu/vectors/3dcoordinates.htm.
Use the above formula to convert spherical coordinates(Polar) to Cartesian and calculate Euclidean distance.
I used the following c# in a console app.
Considering following dummy lat long
double lat_1 = 18.457793 * (Math.PI / 180);
double lon_1 = 73.3951930277778 *(Math.PI/180);
double alt_1 = 270.146;
double lat_2 = 18.4581253333333 * (Math.PI / 180);
double lon_2 = 73.3963755277778 * (Math.PI / 180);
double alt_2 = 317.473;
const Double r = 6376.5 *1000; // Radius of Earth in metres
double x_1 = r * Math.Sin(lon_1) * Math.Cos(lat_1);
double y_1 = r * Math.Sin(lon_1) * Math.Sin(lat_1);
double z_1 = r * Math.Cos(lon_1);
double x_2 = r * Math.Sin(lon_2) * Math.Cos(lat_2);
double y_2 = r * Math.Sin(lon_2) * Math.Sin(lat_2);
double z_2 = r * Math.Cos(lon_2);
double dist = Math.Sqrt((x_2 - x_1) * (x_2 - x_1) + (y_2 - y_1) *
(y_2 - y_1) + (z_2 - z_1) * (z_2 - z_1));
I have a series of points, of right ascension and declination values.
These points correspond to the vertices of a polygon on the surface of a sphere.
What would be the best way to calculate the area enclosed by these points? I would assume that converting the points with an equal-area projection, and then carrying out typical polygonal area calculating on a flat surface would be an appropriate solution.
note: I cannot use custom python libraries. eg pyproj or shapely
Example code (works for latitude longitude, what modifications would be required to enure this works with sky coordinates?)
def reproject(latitude, longitude):
"""Returns the x & y coordinates in metres using a sinusoidal projection"""
from math import pi, cos, radians
earth_radius = 6371009
lat_dist = pi * earth_radius / 180.0
y = [lat * lat_dist for lat in latitude]
x = [long * lat_dist * cos(radians(lat))
for lat, long in zip(latitude, longitude)]
return x, y
def area_of_polygon(x, y):
"""Calculates the area of an arbitrary polygon given its vertices"""
area = 0.0
for i in xrange(-1, len(x)-1):
area += x[i] * (y[i+1] - y[i-1])
return abs(area) / 2.0
dec = [-15.,89.,89.,-15.,-15.]
ra = [105.,105.,285.,285.,105.]
x,y = reproject(dec, ra)
print area_of_polygon(x,y)
One of the ways is to perform a line integral based on Green's Theorem. See below an implementation, and this question for more details.
def polygon_area(lats, lons, algorithm = 0, radius = 6378137):
"""
Computes area of spherical polygon, assuming spherical Earth.
Returns result in ratio of the sphere's area if the radius is specified.
Otherwise, in the units of provided radius.
lats and lons are in degrees.
"""
from numpy import arctan2, cos, sin, sqrt, pi, power, append, diff, deg2rad
lats = np.deg2rad(lats)
lons = np.deg2rad(lons)
# Line integral based on Green's Theorem, assumes spherical Earth
#close polygon
if lats[0]!=lats[-1]:
lats = append(lats, lats[0])
lons = append(lons, lons[0])
#colatitudes relative to (0,0)
a = sin(lats/2)**2 + cos(lats)* sin(lons/2)**2
colat = 2*arctan2( sqrt(a), sqrt(1-a) )
#azimuths relative to (0,0)
az = arctan2(cos(lats) * sin(lons), sin(lats)) % (2*pi)
# Calculate diffs
# daz = diff(az) % (2*pi)
daz = diff(az)
daz = (daz + pi) % (2 * pi) - pi
deltas=diff(colat)/2
colat=colat[0:-1]+deltas
# Perform integral
integrands = (1-cos(colat)) * daz
# Integrate
area = abs(sum(integrands))/(4*pi)
area = min(area,1-area)
if radius is not None: #return in units of radius
return area * 4*pi*radius**2
else: #return in ratio of sphere total area
return area
Please find a somewhat more explicit version (and with many more references and TODOs...) here.
Looks like I can treat ra and dec like lat and long, work out the area on the Earth's surface in m^2, and use this value to convert into an area in sq degrees.
Please let me know if the solution I propose below is flawed:
def reproject(latitude, longitude):
"""Returns the x & y coordinates in metres using a sinusoidal projection"""
from math import pi, cos, radians
earth_radius = 6371009
lat_dist = pi * earth_radius / 180.0
y = [lat * lat_dist for lat in latitude]
x = [long * lat_dist * cos(radians(lat))
for lat, long in zip(latitude, longitude)]
return x, y
def area_of_polygon(x, y):
"""Calculates the area of an arbitrary polygon given its vertices"""
area = 0.0
for i in xrange(-1, len(x)-1):
area += x[i] * (y[i+1] - y[i-1])
return ((abs(area) / 2.0)/5.10100E14) * 41253
dec = [-15.,89.,89.,-15.,-15.]
ra = [105.,105.,285.,285.,105.]
x,y = reproject(dec, ra)
print area_of_polygon(x,y)