So I am working on a Arcpy toolbox in Arcgis that take DEM raster file into a certain treatment.
However I need to clip those image because the original ones are way too big and too long to process.
But thing is, Arcgis clipping tool changes the data type which I cannot use then.
I started looking for codes to do that and it appears that GDAL library might help to clip a geotiff with a shapefile. Here is the code I followed with some minor changes to adapt to my 1-channel DEM: < https://pcjericks.github.io/py-gdalogr-cookbook/raster_layers.html >
from osgeo import gdal, gdalnumeric, ogr, osr
from PIL import Image, ImageDraw
gdal.UseExceptions()
# This function will convert the rasterized clipper shapefile
# to a mask for use within GDAL.
def imageToArray(i):
"""
Converts a Python Imaging Library array to a
gdalnumeric image.
"""
a=gdalnumeric.fromstring(i.tostring(),'b')
a.shape=i.im.size[1], i.im.size[0]
return a
def arrayToImage(a):
"""
Converts a gdalnumeric array to a
Python Imaging Library Image.
"""
i=Image.fromstring('L',(a.shape[1],a.shape[0]),
(a.astype('b')).tostring())
return i
def world2Pixel(geoMatrix, x, y):
"""
Uses a gdal geomatrix (gdal.GetGeoTransform()) to calculate
the pixel location of a geospatial coordinate
"""
ulX = geoMatrix[0]
ulY = geoMatrix[3]
xDist = geoMatrix[1]
yDist = geoMatrix[5]
rtnX = geoMatrix[2]
rtnY = geoMatrix[4]
pixel = int((x - ulX) / xDist)
line = int((ulY - y) / xDist)
return (pixel, line)
#
# EDIT: this is basically an overloaded
# version of the gdal_array.OpenArray passing in xoff, yoff explicitly
# so we can pass these params off to CopyDatasetInfo
#
def OpenArray( array, prototype_ds = None, xoff=0, yoff=0 ):
ds = gdal.Open( gdalnumeric.GetArrayFilename(array) )
if ds is not None and prototype_ds is not None:
if type(prototype_ds).__name__ == 'str':
prototype_ds = gdal.Open( prototype_ds )
if prototype_ds is not None:
gdalnumeric.CopyDatasetInfo( prototype_ds, ds, xoff=xoff, yoff=yoff )
return ds
def histogram(a, bins=range(0,256)):
"""
Histogram function for multi-dimensional array.
a = array
bins = range of numbers to match
"""
fa = a.flat
n = gdalnumeric.searchsorted(gdalnumeric.sort(fa), bins)
n = gdalnumeric.concatenate([n, [len(fa)]])
hist = n[1:]-n[:-1]
return hist
def stretch(a):
"""
Performs a histogram stretch on a gdalnumeric array image.
"""
hist = histogram(a)
im = arrayToImage(a)
lut = []
for b in range(0, len(hist), 256):
# step size
step = reduce(operator.add, hist[b:b+256]) / 255
# create equalization lookup table
n = 0
for i in range(256):
lut.append(n / step)
n = n + hist[i+b]
im = im.point(lut)
return imageToArray(im)
def main( shapefile_path, raster_path ):
# Load the source data as a gdalnumeric array
srcArray = gdalnumeric.LoadFile(raster_path)
# Also load as a gdal image to get geotransform
# (world file) info
srcImage = gdal.Open(raster_path)
geoTrans = srcImage.GetGeoTransform()
# Create an OGR layer from a boundary shapefile
shapef = ogr.Open("%s.shp" % shapefile_path)
lyr = shapef.GetLayer( os.path.split( os.path.splitext( shapefile_path )[0] )[1] )
poly = lyr.GetNextFeature()
# Convert the layer extent to image pixel coordinates
minX, maxX, minY, maxY = lyr.GetExtent()
ulX, ulY = world2Pixel(geoTrans, minX, maxY)
lrX, lrY = world2Pixel(geoTrans, maxX, minY)
# Calculate the pixel size of the new image
pxWidth = int(lrX - ulX)
pxHeight = int(lrY - ulY)
clip = srcArray[ulY:lrY, ulX:lrX]
#
# EDIT: create pixel offset to pass to new image Projection info
#
xoffset = ulX
yoffset = ulY
print "Xoffset, Yoffset = ( %f, %f )" % ( xoffset, yoffset )
# Create a new geomatrix for the image
geoTrans = list(geoTrans)
geoTrans[0] = minX
geoTrans[3] = maxY
# Map points to pixels for drawing the
# boundary on a blank 8-bit,
# black and white, mask image.
points = []
pixels = []
geom = poly.GetGeometryRef()
pts = geom.GetGeometryRef(0)
for p in range(pts.GetPointCount()):
points.append((pts.GetX(p), pts.GetY(p)))
for p in points:
pixels.append(world2Pixel(geoTrans, p[0], p[1]))
rasterPoly = Image.new("L", (pxWidth, pxHeight), 1)
rasterize = ImageDraw.Draw(rasterPoly)
rasterize.polygon(pixels, 0)
mask = imageToArray(rasterPoly)
# Clip the image using the mask
clip = gdalnumeric.choose(mask, \
(clip, 0)).astype(gdalnumeric.uint8)
clip[:,:] = stretch(clip[:,:])
# Save new tiff
#
# EDIT: instead of SaveArray, let's break all the
# SaveArray steps out more explicity so
# we can overwrite the offset of the destination
# raster
#
### the old way using SaveArray
#
# gdalnumeric.SaveArray(clip, "OUTPUT.tif", format="GTiff", prototype=raster_path)
#
###
#
gtiffDriver = gdal.GetDriverByName( 'GTiff' )
if gtiffDriver is None:
raise ValueError("Can't find GeoTiff Driver")
gtiffDriver.CreateCopy( "OUTPUT.tif",
OpenArray( clip, prototype_ds=raster_path, xoff=xoffset, yoff=yoffset )
)
# Save as an 8-bit jpeg for an easy, quick preview
clip = clip.astype(gdalnumeric.uint8)
gdalnumeric.SaveArray(clip, "OUTPUT.jpg", format="JPEG")
gdal.ErrorReset()
if __name__ == '__main__':
main( "shapefile", "DEM.tiff" )
However I got a "shape mismatch ValueError":
<ipython-input-22-32e4e8197a02> in main(shapefile_path, raster_path, region_shapefile_path)
166
167 # Clip the image using the mask
--> 168 clip = gdalnumeric.choose(mask, (clip, 0)).astype(gdalnumeric.uint8)
169
/home/edgar/anaconda3/envs/gis2/lib/python2.7/site-packages/numpy/core/fromnumeric.pyc in choose(a, choices, out, mode)
399
400 """
--> 401 return _wrapfunc(a, 'choose', choices, out=out, mode=mode)
402
403
/home/edgar/anaconda3/envs/gis2/lib/python2.7/site-packages/numpy/core/fromnumeric.pyc in _wrapfunc(obj, method, *args, **kwds)
49 def _wrapfunc(obj, method, *args, **kwds):
50 try:
---> 51 return getattr(obj, method)(*args, **kwds)
52
53 # An AttributeError occurs if the object does not have
ValueError: shape mismatch: objects cannot be broadcast to a single shape
I tried to look around in the code where could it come from and I realized that this part is probably not working as it should:
minX, maxX, minY, maxY = lyr.GetExtent()
ulX, ulY = world2Pixel(geoTrans, minX, maxY)
lrX, lrY = world2Pixel(geoTrans, maxX, minY)
print("ulX, lrX, ulY, lrY : " , ulX, lrX, ulY, lrY) #image pixel coordinates of the shapefile
print(srcArray.shape) #shape of the raster image
clip = srcArray[ ulY:lrY, ulX:lrX] #extracting the shapefile zone from the raster image?
print(clip)
And returns:
('ulX, lrX, ulY, lrY : ', 35487, 37121, 3844, 5399)
(5041, 5041)
[]
Seems that those indexes are at of bounds (but strangely python doesn't bother that much) and nothing is copied.
So I tried to change a bit the code to get the "real" pixel value corresponding to the area I wish to extract by using a shapefile corresponding to my total raster image:
#shapefile corresponding to the whole raster image
region_shapef = ogr.Open("%s.shp" % region_shapefile_path)
region_lyr = region_shapef.GetLayer( os.path.split( os.path.splitext( region_shapefile_path )[0] )[1] )
RminX, RmaxX, RminY, RmaxY = region_lyr.GetExtent()
RulX, RulY = world2Pixel(geoTrans, RminX, RmaxY)
RlrX, RlrY = world2Pixel(geoTrans, RmaxX, RminY)
#linear regression to find the equivalent pixel values of the clipping zone
pX = float(srcArray.shape[1])/(RlrX - RulX)
X0 = -(RulX*pX)
pY = float(srcArray.shape[0])/(RlrY - RulY)
Y0 = -(RulY*pY)
idXi = int(ulX*pX+X0)
idXf = int(lrX*pX+X0)
idYi = int(ulY*pY+Y0)
idYf = int(lrY*pY+Y0)
clip = srcArray[idYi:idYf, idXi:idXf]
print(clip)
Returns an array that really extracted values:
[[169.4 171.3 173.7 ... 735.6 732.8 729.7]
[173.3 176.4 179.9 ... 734.3 731.5 728.7]
[177.8 182. 186.5 ... 733.1 730.3 727.5]
...
[ 73.3 77.5 83. ... 577.4 584.9 598.1]
[ 72.8 76.5 81.5 ... 583.1 593. 606.2]
[ 71.3 74.7 79. ... 588.9 599.1 612.3]]
Though I still have that goddamn:
<ipython-input-1-d7714555354e> in main(shapefile_path, raster_path, region_shapefile_path)
170
171 # Clip the image using the mask
--> 172 clip = gdalnumeric.choose(mask, (clip, 0)).astype(gdalnumeric.uint8)
173
174 # This image has 3 bands so we stretch each one to make them
/home/edgar/anaconda3/envs/gis2/lib/python2.7/site-packages/numpy/core/fromnumeric.pyc in choose(a, choices, out, mode)
399
400 """
--> 401 return _wrapfunc(a, 'choose', choices, out=out, mode=mode)
402
403
/home/edgar/anaconda3/envs/gis2/lib/python2.7/site-packages/numpy/core/fromnumeric.pyc in _wrapfunc(obj, method, *args, **kwds)
49 def _wrapfunc(obj, method, *args, **kwds):
50 try:
---> 51 return getattr(obj, method)(*args, **kwds)
52
53 # An AttributeError occurs if the object does not have
ValueError: shape mismatch: objects cannot be broadcast to a single shape
Am I missing or misunderstanding something about it? I really start to lack of ideas so if someone got an idea please that would be appreciated.
Otherwise if you know another way to clip my DEM without altering it I'm fine as well.
You can achieve that in a much more simple way using gdal.Warp() and a shapefile as a cutline
from osgeo import gdal
input_raster = "path/to/yourDEM.tif"
# or as an alternative if the input is already a gdal raster object you can use that gdal object
input_raster=gdal.Open("path/to/yourDEM.tif")
input_shape = "path/to/yourShapefile.shp" # or any other format
output_raster="path/to/outputDEM.tif" #your output raster file
ds = gdal.Warp(output_raster,
input_raster,
format = 'GTiff',
cutlineDSName = input_shape, # or any other file format
cutlineWhere="FIELD = 'whatever'" # optionally you can filter your cutline (shapefile) based on attribute values
dstNodata = -9999) # select the no data value you like
ds=None #do other stuff with ds object, it is your cropped dataset. in this case we only close the dataset.
Related
Here I have some code that can vertically and horizontally shift images so that a specific feature can align (credits to https://stackoverflow.com/a/24769222/15016884):
def cross_image(im1, im2):
im1_gray = np.sum(im1.astype('float'), axis=2)
im2_gray = np.sum(im2.astype('float'), axis=2)
im1_gray -= np.mean(im1_gray)
im2_gray -= np.mean(im2_gray)
return signal.fftconvolve(im1_gray, im2_gray[::-1,::-1], mode='same')
corr_img_null = cross_image(cloud1,cloud1)
corr_img = cross_image(cloud1,cloud2)
y0, x0 = np.unravel_index(np.argmax(corr_img_null), corr_img_null.shape)
y, x = np.unravel_index(np.argmax(corr_img), corr_img.shape)
ver_shift = y0-y
hor_shift = x0-x
print('horizontally shifted', hor_shift)
print('vertically shifted', ver_shift)
#defining the bounds of the part of the images I'm actually analyzing
xstart = 100
xstop = 310
ystart = 50
ystop = 200
crop_cloud1 = cloud1[ystart:ystop, xstart:xstop]
crop_cloud2 = cloud2[ystart:ystop, xstart:xstop]
crop_cloud2_shift = cloud2[ystart+ver_shift:ystop+ver_shift, xstart+hor_shift:xstop+hor_shift]
plot_pos = plt.figure(5)
plt.title('image 1')
plt.imshow(crop_cloud1)
plot_pos = plt.figure(6)
plt.title('image 2')
plt.imshow(crop_cloud2)
plot_pos = plt.figure(7)
plt.title('Shifted image 2 to align with image 1')
plt.imshow(crop_cloud2_shift)
Here are the results:
Now, I want to work with the example shown below, where rotations in addition to translations will be needed to align the features in my image.
Here is my code for that: The idea is to convolve each possible configuration of image 2 for every angle from -45 to 45 (for my application, this angle is not likely to be exceeded) and find at which coordinates and rotation angle the convolution is maximized.
import cv2
def rotate(img, theta):
(rows, cols) = img.shape[:2]
M = cv2.getRotationMatrix2D((cols / 2, rows / 2), theta, 1)
res = cv2.warpAffine(img, M, (cols, rows))
return res
#testing all rotations of image 2
corr_bucket = []
for i in range(-45,45):
rot_img = rotate(bolt2,i)
corr_img = cross_image(bolt1,rot_img)
corr_bucket.append(corr_img)
corr_arr = np.asarray(corr_bucket)
corr_img_null = cross_image(bolt1,bolt1)
y0, x0 = np.unravel_index(np.argmax(corr_img_null), corr_img_null.shape)
r_index, y1, x1 = np.unravel_index(np.argmax(corr_arr), corr_arr.shape)
r = -45+r_index
ver_shift = y0-y
hor_shift = x0-x
ver_shift_r = y0-y1
hor_shift_r = x0-x1
#What parts of the image do you want to analyze
xstart = 200
xstop = 300
ystart = 100
ystop = 200
crop_bolt1 = bolt1[ystart:ystop, xstart:xstop]
crop_bolt2 = bolt2[ystart:ystop, xstart:xstop]
rot_bolt2 = rotate(bolt2,r)
shift_rot_bolt2 = rot_bolt2[ystart+ver_shift_r:ystop+ver_shift_r, xstart+hor_shift_r:xstop+hor_shift_r]
plot_1 = plt.figure(9)
plt.title('image 1')
plt.imshow(crop_bolt1)
plot_2 = plt.figure(10)
plt.title('image 2')
plt.imshow(crop_bolt2)
plot_3 = plt.figure(11)
plt.title('Shifted and rotated image 2 to align with image 1')
plt.imshow(shift_rot_bolt2)
Unfortunately, from the very last line, I get the error ValueError: zero-size array to reduction operation minimum which has no identity. I'm kind of new to python so I don't really know what this means or why my approach isn't working. I have a feeling that my error is somewhere in unraveling corr_arr because the x, y and r values it returns I can already see, just by estimating, would not make the lightning bolts align. Any advice?
The issue came from feeding in the entire rotated image into scipy.signal.fftconvolve. Crop a part of image2 after rotating to use as a "probe image" (crop your unrotated image 1 in the same way), and the code I have written in my question works fine.
My goal is to align a shapefile to a raster basemap, and assign 1 to the cells that overlap and 0 to the ones that don't, eventually returning an array that contains lat, lon, time, and the binary variable (1/0).
Here's the plan: 1) create raster of region from array, 2) rasterize polygon shapefiles, 3) align rasterized shapefiles with base raster, 4) pixels that overlap will be assigned 1 and those that don't will be 0, 5) convert rasters to array.
I've been able to do steps 1 & 2 (see code below), but I've been stuck on step 3 for a long time. How do I align the two rasters?
You can find the files here:
https://www.dropbox.com/sh/pecptfepac18s2y/AADbxFkKWlLqMdiHh-ICt4UYa?dl=0
Here's the code I used to create a flat grid of BC as basemap:
import gdal, osr
import numpy as np
#define parameters
#units = km
grid_size = 5
BC_width = 700
BC_length = 1800
def array2raster(newRasterfn,rasterOrigin,pixelWidth,pixelHeight,array):
cols = array.shape[1]
rows = array.shape[0]
originX = rasterOrigin[0]
originY = rasterOrigin[1]
driver = gdal.GetDriverByName('GTiff')
outRaster = driver.Create(newRasterfn, cols, rows, 1, gdal.GDT_Byte)
outRaster.SetGeoTransform((originX, pixelWidth, 0, originY, 0, pixelHeight))
outband = outRaster.GetRasterBand(1)
outband.WriteArray(array)
outRasterSRS = osr.SpatialReference()
outRasterSRS.ImportFromEPSG(4326)
outRaster.SetProjection(outRasterSRS.ExportToWkt())
outband.FlushCache()
def main(newRasterfn,rasterOrigin,pixelWidth,pixelHeight,array):
reversed_arr = array[::-1] # reverse array so the tif looks like the array
array2raster(newRasterfn,rasterOrigin,pixelWidth,pixelHeight,reversed_arr) # convert array to raster
if __name__ == "__main__":
array = np.zeros([int(BC_length/grid_size),int(BC_width/grid_size)]) #140x360
for i in range(1,100):
array[i] = 100
rasterOrigin = (-139.72938, 47.655534) #lower left corner of raster
newRasterfn = '/temp/test.tif'
cols = array.shape[1] #shape of an array (aka # of elements in each dimension)
rows = array.shape[0]
originX = rasterOrigin[0]
originY = rasterOrigin[1]
pixelWidth = 5
pixelHeight = 5
Here's the code I used to rasterize polygon shapefiles
import ogr, gdal, osr
output_raster = '/testdata/poly.tif'
shapefile = "/testdata/20180808.shp"
def main(shapefile):
#making the shapefile as an object.
input_shp = ogr.Open(shapefile)
#getting layer information of shapefile.
shp_layer = input_shp.GetLayer()
#pixel_size determines the size of the new raster.
#pixel_size is proportional to size of shapefile.
pixel_size = 0.1
#get extent values to set size of output raster.
x_min, x_max, y_min, y_max = shp_layer.GetExtent()
#calculate size/resolution of the raster.
x_res = int((x_max - x_min) / pixel_size)
y_res = int((y_max - y_min) / pixel_size)
#get GeoTiff driver by
image_type = 'GTiff'
driver = gdal.GetDriverByName(image_type)
#passing the filename, x and y direction resolution, no. of bands, new raster.
new_raster = driver.Create(output_raster, x_res, y_res, 1, gdal.GDT_Byte)
#transforms between pixel raster space to projection coordinate space.
new_raster.SetGeoTransform((x_min, pixel_size, 0, y_min, 0, pixel_size))
#get required raster band.
band = new_raster.GetRasterBand(1)
#assign no data value to empty cells.
no_data_value = -9999
band.SetNoDataValue(no_data_value)
band.FlushCache()
#main conversion method
gdal.RasterizeLayer(new_raster, [1], shp_layer, burn_values=[255])
#adding a spatial reference
new_rasterSRS = osr.SpatialReference()
new_rasterSRS.ImportFromEPSG(4326)
new_raster.SetProjection(new_rasterSRS.ExportToWkt())
return output_raster
I'm doing everything in Python as I don't have access or funding to paid GIS software. I'm totally new to geospatial data processing... not sure if I'm taking the right approach. Any help would be amazing.
Checkout 'rasterio.mask.mask' from the rasterio library. I think it will help.
Question:
How can I programmatically return a raster that is the difference of two (differently sized) red bands?
i.e.
gdal_calc.py -A 'WARPED.tif' -B 'DSC_1636.tif' --outfile = 'dif.tif' --calc = "A-B"
QGIS raster calculator performs this function just fine. However, the previous code returns the following error.
Exception: Error! Dimensions of file DSC_1636.tif (7380, 4928) are different from other files (7743, 5507). Cannot proceed
I am currently under the impression I should read in the rasters using a defined extent, created by finding the overlap as shown below, but I am still not able to make this work.
# Subtract two rasters of different dimensions
# Pixel coordinates define overlap
import os, sys
from PIL import Image
from osgeo import gdal, ogr, osr
gdal.UseExceptions()
# Use PIL to get information from images
im1 = Image.open('DSC_0934-warped.tif')
print('warped image size is %s ' % str(im1.size))
im2 = Image.open('DSC_1636.png')
print('initial image (image 2) size is %s' % str(im2.size))
warped image size is (7743, 5507)
initial image (image 2) size is (7380, 4928)
# Use GDAL to get information about images
def get_extent(fn):
'''Returns min_x, max_y, max_x, min_y'''
ds = gdal.Open(fn)
gt = ds.GetGeoTransform()
return (gt[0], gt[3], gt[0] + gt[1] * ds.RasterXSize,
gt[3] + gt[5] * ds.RasterYSize)
print('extent of warped.tif is %s' % str(get_extent('DSC_0934-warped.tif')))
print('extent of 1636.png is %s' % str(get_extent('DSC_1636.png')))
extent of warped.tif is (-375.3831214210602, 692.5167764068751, 7991.3588371542955, -5258.102875649754)
extent of 1636.png is (0.0, 0.0, 7380.0, 4928.0)
r1 = get_extent('DSC_0934-warped.tif')
r2 = get_extent('DSC_1636.png')
# Get left, top, right, bottom of dataset's bounds in pixel coordinates
intersection = [max(r1[0], r2[0]),
min(r1[1], r2[1]),
min(r1[2], r2[2]),
max(r1[3], r2[3])]
print('checking for overlap')
if (intersection[2] < intersection[0]) or (intersection[1] > intersection[3]):
intersection = None
print('no overlap')
else:
print('intersection overlaps at: %s' % intersection)
checking for overlap
intersection overlaps at: [0.0, 0.0, 7380.0, 4928.0]
The most straight forward answer is to read in the images as an array of defined dimensions.
Without reposting the code above used to check where the overlap is, the solution can be had with the following additions. (Thank you #Val)
# Get the data
ds1_src = gdal.Open( "DSC_1636.png" )
ds2_src = gdal.Open( "DSC_0934-warped.tif")
ds1_bnd = ds1_src.GetRasterBand(1).ReadAsArray(xoff=0, yoff=0, win_xsize=7380, win_ysize=4928)
ds2_bnd = ds2_src.GetRasterBand(1).ReadAsArray(xoff=0, yoff=0, win_xsize=7380, win_ysize=4928)
# Do the maths...
data_out = ds2_bnd - ds1_bnd
#Write the out file
driver = gdal.GetDriverByName("GTiff")
dsOut = driver.Create("out.tiff", 7380, 4928, 1, GDT_Byte)
CopyDatasetInfo(ds1_src,dsOut)
bandOut=dsOut.GetRasterBand(1)
BandWriteArray(bandOut, data_out)
#Close the datasets
ds1_src = None
ds2_src = None
ds1_bnd = None
ds2_bnd = None
bandOut = None
dsOut = None
This function receives a list of numpy arrays that consist of cropped parts of an image. The crops are all the same size, except for the right-most and bottom-most images which might be of smaller size.
predictions[2] would return the 3rd sub-image that was cropped from the original image. Each crop is a numpy array. There are WxH crops, enumerated from left to right, top to bottom (so if there are 4 sub-images constituting the width, the 5th image in predictions would be the first sub-image on the left from the 2nd row of sub-images).
crops contains the necessary information to find number of horizontal and vertical images that will constitute the reconstructed images. crops[2][3] will contain the 3rd from the top, 4th from the left image cropped.
The images contained by crops are of smaller dimension than the ones in predictions (I am basically making a model that increases the resolution of images). The reconstructed image if from the images in predictions, arranged in the same order as the ones in crops.
def reconstruct(predictions, crops):
if len(crops) != 0:
print("use crops")
# TODO: properly extract the size of the full image
width_length = 0
height_length = 0
full_image = np.empty(shape=(height_length, width_length))
print(full_image.shape)
# TODO: properly merge the crops back into a single image
for height in range(len(predictions[0])):
for width in range(len(predictions)):
# concatenate here
print(height, width)
return full_image
I was going to use numpy.concatenate, but according to other answers I've seen on SO it wouldn't be an efficient way of doing it (apparently numpy will just recreate a new variable in memory, copy the old one, and add the new data, etc.). So now I'm left wondering how to properly merge my multiple images into a single image. The current idea I was going for was to create a python list of the proper shape and progressively fill it with each numpy array's data, but even that I'm not sure if it's the proper idea.
Here is more or less the kind of bunch of images I'm trying to concatenate into a single image:
Here is the expected result:
And to help you out with understanding what more might be available to you, here is some more code:
def predict(args):
model = load_model(save_dir + '/' + args.model)
image = skimage.io.imread(tests_path + args.image)
predictions = []
images = []
crops = seq_crop(image) # crops into multiple sub-parts the image based on 'input_' constants
for i in range(len(crops)): # amount of vertical crops
for j in range(len(crops[0])): # amount of horizontal crops
current_image = crops[i][j]
images.append(current_image)
# Hack because GPU can only handle one image at a time
input_img = (np.expand_dims(images[p], 0)) # Add the image to a batch where it's the only member
predictions.append(model.predict(input_img)[0]) # returns a list of lists, one for each image in the batch
return predictions, image, crops
# adapted from: https://stackoverflow.com/a/52463034/9768291
def seq_crop(img):
"""
To crop the whole image in a list of sub-images of the same size.
Size comes from "input_" variables in the 'constants' (Evaluation).
Padding with 0 the Bottom and Right image.
:param img: input image
:return: list of sub-images with defined size
"""
width_shape = ceildiv(img.shape[1], input_width)
height_shape = ceildiv(img.shape[0], input_height)
sub_images = [] # will contain all the cropped sub-parts of the image
for j in range(height_shape):
horizontal = []
for i in range(width_shape):
horizontal.append(crop_precise(img, i*input_width, j*input_height, input_width, input_height))
sub_images.append(horizontal)
return sub_images
def crop_precise(img, coord_x, coord_y, width_length, height_length):
"""
To crop a precise portion of an image.
When trying to crop outside of the boundaries, the input to padded with zeros.
:param img: image to crop
:param coord_x: width coordinate (top left point)
:param coord_y: height coordinate (top left point)
:param width_length: width of the cropped portion starting from coord_x
:param height_length: height of the cropped portion starting from coord_y
:return: the cropped part of the image
"""
tmp_img = img[coord_y:coord_y + height_length, coord_x:coord_x + width_length]
return float_im(tmp_img) # From [0,255] to [0.,1.]
# from https://stackoverflow.com/a/17511341/9768291
def ceildiv(a, b):
"""
To get the ceiling of a division
:param a:
:param b:
:return:
"""
return -(-a // b)
if __name__ == '__main__':
preds, original, crops = predict(args) # returns the predictions along with the original
# TODO: reconstruct image
enhanced = reconstruct(preds, crops) # reconstructs the enhanced image from predictions
EDIT:
The answer worked. Here is the version I've used:
# adapted from https://stackoverflow.com/a/52733370/9768291
def reconstruct(predictions, crops):
# unflatten predictions
def nest(data, template):
data = iter(data)
return [[next(data) for _ in row] for row in template]
predictions = nest(predictions, crops)
H = np.cumsum([x[0].shape[0] for x in predictions])
W = np.cumsum([x.shape[1] for x in predictions[0]])
D = predictions[0][0]
recon = np.empty((H[-1], W[-1], D.shape[2]), D.dtype)
for rd, rs in zip(np.split(recon, H[:-1], 0), predictions):
for d, s in zip(np.split(rd, W[:-1], 1), rs):
d[...] = s
return recon
The most convenient is probably np.block
import numpy as np
from scipy import misc
import Image
# get example picture
data = misc.face()
# chop it up
I, J = map(np.arange, (200, 200), data.shape[:2], (200, 200))
chops = [np.split(row, J, axis=1) for row in np.split(data, I, axis=0)]
# do something with the bits
predictions = [chop-(i+j)*(chop>>3) for j, row in enumerate(chops) for i, chop in enumerate(row)]
# unflatten predictions
def nest(data, template):
data = iter(data)
return [[next(data) for _ in row] for row in template]
pred_lol = nest(predictions, chops)
# almost builtin reconstruction
def np_block_2D(chops):
return np.block([[[x] for x in row] for row in chops])
recon = np_block_2D(pred_lol)
Image.fromarray(recon).save('demo.png')
Reconstructed manipulated image:
But we can do faster than that by avoiding intermediary arrays. Instead, we copy into a preallocated array:
def speed_block_2D(chops):
H = np.cumsum([x[0].shape[0] for x in chops])
W = np.cumsum([x.shape[1] for x in chops[0]])
D = chops[0][0]
recon = np.empty((H[-1], W[-1], D.shape[2]), D.dtype)
for rd, rs in zip(np.split(recon, H[:-1], 0), chops):
for d, s in zip(np.split(rd, W[:-1], 1), rs):
d[...] = s
return recon
Timings, also including a generalized ND-ready variant of each method:
numpy 2D: 0.991 ms
prealloc 2D: 0.389 ms
numpy general: 1.021 ms
prealloc general: 0.448 ms
Code for general case and timings:
def np_block(chops):
d = 0
tl = chops
while isinstance(tl, list):
tl = tl[0]
d += 1
if d < tl.ndim:
def adjust_depth(L):
if isinstance(L, list):
return [adjust_depth(l) for l in L]
else:
ret = L
for j in range(d, tl.ndim):
ret = [ret]
return ret
chops = adjust_depth(chops)
return np.block(chops)
def speed_block(chops):
def line(src, i):
while isinstance(src, list):
src = src[0]
return src.shape[i]
def hyper(src, i):
src = iter(src)
fst = next(src)
if isinstance(fst, list):
res, dtype, szs = hyper(fst, i+1)
szs.append([res[i], *(line(s, i) for s in src)])
res[i] = sum(szs[-1])
return res, dtype, szs
res = np.array(fst.shape)
szs = [res[i], *(s.shape[i] for s in src)]
res[i] = sum(szs)
return res, fst.dtype, [szs]
shape, dtype, szs = hyper(chops, 0)
recon = np.empty(shape, dtype)
def cpchp(dst, src, i, szs=None):
szs = np.array(hyper(src, i)[2]) if szs is None else szs
dst = np.split(dst, np.cumsum(szs[-1][:-1]), i)
if isinstance(src[0], list):
szs = szs[:-1]
for ds, sr in zip(dst, src):
cpchp(ds, sr, i+1, szs)
szs = None
else:
for ds, sr in zip(dst, src):
ds[...] = sr
cpchp(recon, chops, 0, np.array(szs))
return recon
from timeit import timeit
T = (timeit(lambda: speed_block(pred_lol), number=1000),
timeit(lambda: np_block(pred_lol), number=1000),
timeit(lambda: speed_block_2D(pred_lol), number=1000),
timeit(lambda: np_block_2D(pred_lol), number=1000))
assert (np.all(speed_block(pred_lol)==np_block(pred_lol)) and
np.all(speed_block_2D(pred_lol)==np_block(pred_lol)) and
np.all(speed_block(pred_lol)==np_block_2D(pred_lol)))
print(f"""
numpy 2D: {T[3]:10.3f} ms
prealloc 2D: {T[2]:10.3f} ms
numpy general: {T[1]:10.3f} ms
prealloc general: {T[0]:10.3f} ms
""")
When using the OGR library or GDAL library with Python script, is it possible to increase the extent of a vector layer without actually adding new data points? In my specific case, I would like to increase the extent of vector layers associated with gpx files so that when I convert them to rasters they all have the same pixel matrix.
EDIT: An attempt of mine to use gdal.Rasterize does not produce a "tiff" file, nor does it cause an error to be reported:
import os
import gdal
import ogr
import math
os.chdir(r'C:\Users\pipi\Documents\Rogaine\Tarlo\gpx') #folder containing gpx files
vector_fn = '6_hour_Autumngaine_w_Tom_Elle.gpx' #filename of input gpxfile
pixel_size = 20 #units are in m if gpx file is left in wgs84
raster_fn = '0011a.tif' # Filename of the raster Tiff that will be created
driver = ogr.GetDriverByName('GPX')
source_ds = driver.Open(vector_fn, 0)
source_layer = source_ds.GetLayer('track_points') #returns the 'track points' layer of the data source
SR = source_layer.GetSpatialRef().ExportToWkt()
#_______USING VALUES FROM THE FILE___________
x_min1, x_max1, y_min1, y_max1 = source_layer.GetExtent()
pixel_sizey = pixel_size/(111.2*math.pow(10,3)) #determines an approximate x and y size because of geographic coordinates.
pixel_sizex = pixel_size/(math.cos(((y_max1 + y_min1)/2)*(math.pi/180))*111.2*math.pow(10,3))
print (pixel_sizey, pixel_sizex)
x_res = int((x_max1 - x_min1) / pixel_sizex)
y_res = int((y_max1 - y_min1) / pixel_sizey)
print (x_res, y_res)
layer_list = ['track_points']
gdal.Rasterize(raster_fn, vector_fn, format='GTiff', outputBounds=[x_min1, y_min1, x_max1, y_max1], outputSRS=SR, xRes=x_res, yRes=y_res, burnValues=[1], layers=layer_list)
target_ds = None
vector_fn = None
source_layer = None
source_ds = None
You need to pass options=gdal.RasterizeOptions(format='GTiff', outputBounds=[x_min1, y_min1, x_max1, y_max1], outputSRS=SR, xRes=x_res, yRes=y_res, burnValues=[1], layers=layer_list) instead of passing the individual kwargs directly. Otherwise, they will be ignored, and the command won't do what you intend. See Link and Link for details and links to the source code (often useful given the terse documentation).
I was unable to find a method to change the extent of the vector layer. However, I was able to write a python Function that uses gdal.RasterizeLayer() to produce a raster with an extent much larger than the original vector layer. The code for this function is:
import os
import gdal
import ogr
def RasterizeLarge(name, layer, extent, pixel_size):
"""Used to rasterize a layer where the raster extent is much larger than the layer extent
Arguments:
name -- (string) filename without extension of raster to be produced
layer -- (vector layer object) vector layer containing the data to be rasterized (tested with point data)
extent -- (list: x_min, x_max, y_min, y_max) extent of raster to be produced
pixel_size -- (list: x_pixel_size, y_pixel_size) 1 or 2 pixel different pixel sizes may be sent
"""
if isinstance(pixel_size, (list, tuple)):
x_pixel_size = pixel_size[0]
y_pixel_size = pixel_size[1]
else:
x_pixel_size = y_pixel_size = pixel_size
x_min, x_max, y_min, y_max = extent
# determines the x and y resolution of the file (lg = large)
x_res_lg = int((x_max - x_min) / x_pixel_size)+2
y_res_lg = int((y_max - y_min) / y_pixel_size)+2
if x_res_lg > 1 and y_res_lg > 1:
pass
else:
print ('Your pixel size is larger than the extent in one dimension or more')
return
x_min_sm, x_max_sm, y_min_sm, y_max_sm = layer.GetExtent()
if x_min_sm > x_min and x_max_sm < x_max and y_min_sm > y_min and y_max_sm < y_max:
pass
else:
print ('The extent of the layer is in one or more parts outside of the extent provided')
return
nx = int((x_min_sm - x_min)/x_pixel_size) #(number of pixels between main raster origin and minor raster)
ny = int((y_max - y_max_sm)/y_pixel_size)
x_res_sm = int((x_max_sm - x_min_sm) / x_pixel_size)+2
y_res_sm = int((y_max_sm - y_min_sm) / y_pixel_size)+2
#determines upper left corner of small layer raster
x_min_sm = x_min + nx * x_pixel_size
y_max_sm = y_max - ny * y_pixel_size
#______Creates a temporary raster file for the small raster__________
try:
# create the target raster file with 1 band
sm_ds = gdal.GetDriverByName('GTiff').Create('tempsmall.tif', x_res_sm, y_res_sm, 1, gdal.GDT_Byte)
sm_ds.SetGeoTransform((x_min_sm, x_pixel_size, 0, y_max_sm, 0, -y_pixel_size))
sm_ds.SetProjection(layer.GetSpatialRef().ExportToWkt())
gdal.RasterizeLayer(sm_ds, [1], layer, burn_values=[1])
sm_ds.FlushCache()
#______Gets data from the new raster in the form of an array________
in_band = sm_ds.GetRasterBand(1)
in_band.SetNoDataValue(0)
sm_data = in_band.ReadAsArray()
finally:
sm_ds = None #flushes data from memory. Without this you often get an empty raster.
#_____Creates an output file with the provided name and extent that contains the small raster.
name = name + '.tif'
try:
lg_ds = gdal.GetDriverByName('GTiff').Create(name, x_res_lg, y_res_lg, 1, gdal.GDT_Byte)
if lg_ds is None:
print 'Could not create tif'
return
else:
pass
lg_ds.SetProjection(layer.GetSpatialRef().ExportToWkt())
lg_ds.SetGeoTransform((x_min, x_pixel_size, 0.0, y_max, 0.0, -y_pixel_size))
lg_band = lg_ds.GetRasterBand(1)
lg_data = in_band.ReadAsArray()
lg_band.WriteArray(sm_data, xoff = nx, yoff = ny)
lg_band.SetNoDataValue(0)
lg_band.FlushCache()
lg_band.ComputeStatistics(False)
lg_band = None
finally:
del lg_ds, lg_band, in_band
os.remove('tempsmall.tif')
return