Develop Reference

How to code up an image stitching software for these 'simple' images?

TLDR:
Need help trying to calculate overlap region between 2 graphs.
So I'm trying to stitch these 2 images:
Since I know that the images I will be stitching definitely come from the same image, I feel that I should be able to code this up myself. Using libraries like OpenCV feels a little like overkill for me for this task.
My current idea is that I can simplify this task by doing the following steps for each image:
Load image using PIL
Convert image to black and white (PIL image mode “L”)
[Optional: crop images to overlapping region by inspection by eye]
Create vector row_sum, which is a sum of each row
[Optional: log row_sum, to reduce the size of values we're working with]
Plot row_sum.
This would reduce the (potentially) (3*2)-dimensional problem, with 3 RGB channels for each pixel on the 2D image to a (1*2)-D problem with the black and white pixel for the 2D image instead. Then, summing across the rows reduces this to a 1D problem.
I used the following code to implement the above:
import matplotlib.pyplot as plt
import numpy as np
from PIL import Image
class Stitcher():
def combine_2(self, img1, img2):
# thr1, thr2 = self.get_cropped_bw(img1, 115, img2, 80)
thr1, thr2 = self.get_cropped_bw(img1, 0, img2, 0)
row_sum1 = np.log(thr1.sum(1))
row_sum2 = np.log(thr2.sum(1))
self.plot_4x4(thr1, thr2, row_sum1, row_sum2)
def get_cropped_bw(self, img1, img1_keep_from, img2, img2_keep_till):
im1 = Image.open(img1).convert("L")
im2 = Image.open(img2).convert("L")
data1 = (np.array(im1)[img1_keep_from:]
if img1_keep_from != 0 else np.array(im1))
data2 = (np.array(im2)[:img2_keep_till]
if img2_keep_till != 0 else np.array(im2))
return data1, data2
def plot_4x4(self, thr1, thr2, row_sum1, row_sum2):
fig, ax = plt.subplots(2, 2, sharey="row", constrained_layout=True)
ax[0, 0].imshow(thr1, cmap="Greys")
ax[0, 1].imshow(thr2, cmap="Greys")
ax[1, 0].plot(row_sum1, "k.")
ax[1, 1].plot(row_sum2, "r.")
ax[1, 0].set(
xlabel="Index Value",
ylabel="Row Sum",
)
plt.show()
imgs = (r"combine\imgs\test_image_part_1.jpg",
r"combine\imgs\test_image_part_2.jpg")
s = Stitcher()
s.combine_2(*imgs)
This gave me this graph:
(I've added in those yellow boxes, to indicate the overlap regions.)
This is the bit I'm stuck at. I want to find exactly:
the index value of the left-side of the yellow box for the 1st image and
the index value of the right-side of the yellow box for the 2nd image.
I define the overlap region as the longest range for which the end of the 1st graph 'matches' the start of the 2nd graph. For the method to find the overlap region, what should I do if the row sum values aren't exactly the same (what if one is the other scaled by some factor)?
I feel like this could be a problem that could use dot products to find the similarity between the 2 graphs? But I can't think of how to implement this.

I had a lot more fun with this than I expected. I wrote this using opencv, but that's just to load and show the image. Everything else is done with numpy so swapping this to PIL shouldn't be too difficult.
I'm using a brute-force matcher. I also wrote a random-start hillclimber that runs in much less time, but I can't guarantee it'll find the correct answer since the gradient space isn't smooth. I won't include it in my code since it's long and janky, but if you really need the time efficiency I can add it back in later.
I added a random crop and some salt and pepper noise to the images to test for robustness.
The brute-force matcher operates on the idea that we don't know which section of the two images overlap, so we need to convolve the smaller image over the larger image from left to right, top to bottom. This means our search space is:
horizontal = small_width + big_width
vertical = small_height + big_height
area = horizontal * vertical
This will grow very quickly with image size. I motivate the algorithm by giving it points for having a larger overlap, but it loses more points for having differences in color for the overlapped area.
Here are some pictures from an execution of this program
import cv2
import numpy as np
import random
# randomly snips edges
def randCrop(image, maxMargin):
c = [random.randint(0,maxMargin) for a in range(4)];
return image[c[0]:-c[1], c[2]:-c[3]];
# adds noise to image
def saltPepper(image, minNoise, maxNoise):
h,w = image.shape;
randNum = random.randint(minNoise, maxNoise);
for a in range(randNum):
x = random.randint(0, w-1);
y = random.randint(0, h-1);
image[y,x] = random.randint(0, 255);
return image;
# evaluate layout
def getScore(one, two):
# do raw subtraction
left = one - two;
right = two - one;
sub = np.minimum(left, right);
return np.count_nonzero(sub);
# return 2d random position within range
def randPos(img, big_shape):
th,tw = big_shape;
h,w = img.shape;
x = random.randint(0, tw - w);
y = random.randint(0, th - h);
return [x,y];
# overlays small image onto big image
def overlay(small, big, pos):
# unpack
h,w = small.shape;
x,y = pos;
# copy and place
copy = big.copy();
copy[y:y+h, x:x+w] = small;
return copy;
# calculates overlap region
def overlap(one, two, pos_one, pos_two):
# unpack
h1,w1 = one.shape;
h2,w2 = two.shape;
x1,y1 = pos_one;
x2,y2 = pos_two;
# set edges
l1 = x1;
l2 = x2;
r1 = x1 + w1;
r2 = x2 + w2;
t1 = y1;
t2 = y2;
b1 = y1 + h1;
b2 = y2 + h2;
# go
left = max(l1, l2);
right = min(r1, r2);
top = max(t1, t2);
bottom = min(b1, b2);
return [left, right, top, bottom];
# wrapper for overlay + getScore
def fullScore(one, two, pos_one, pos_two, big_empty):
# check positions
x,y = pos_two;
h,w = two.shape;
th,tw = big_empty.shape;
if y+h > th or x+w > tw or x < 0 or y < 0:
return -99999999;
# overlay
temp_one = overlay(one, big_empty, pos_one);
temp_two = overlay(two, big_empty, pos_two);
# get overlap
l,r,t,b = overlap(one, two, pos_one, pos_two);
temp_one = temp_one[t:b, l:r];
temp_two = temp_two[t:b, l:r];
# score
diff = getScore(temp_one, temp_two);
score = (r-l) * (b-t);
score -= diff*2;
return score;
# do brute force
def bruteForce(one, two):
# calculate search space
# unpack size
h,w = one.shape;
one_size = h*w;
h,w = two.shape;
two_size = h*w;
# small and big
if one_size < two_size:
small = one;
big = two;
else:
small = two;
big = one;
# unpack size
sh, sw = small.shape;
bh, bw = big.shape;
total_width = bw + sw * 2;
total_height = bh + sh * 2;
# set up empty images
empty = np.zeros((total_height, total_width), np.uint8);
# set global best
best_score = -999999;
best_pos = None;
# start scrolling
ybound = total_height - sh;
xbound = total_width - sw;
for y in range(ybound):
print("y: " + str(y) + " || " + str(empty.shape));
for x in range(xbound):
# get score
score = fullScore(big, small, [sw,sh], [x,y], empty);
# show
# prog = overlay(big, empty, [sw,sh]);
# prog = overlay(small, prog, [x,y]);
# cv2.imshow("prog", prog);
# cv2.waitKey(1);
# compare
if score > best_score:
best_score = score;
best_pos = [x,y];
print("best_score: " + str(best_score));
return best_pos, [sw,sh], small, big, empty;
# do a step of hill climber
def hillStep(one, two, best_pos, big_empty, step):
# make a step
new_pos = best_pos[1][:];
new_pos[0] += step[0];
new_pos[1] += step[1];
# get score
return fullScore(one, two, best_pos[0], new_pos, big_empty), new_pos;
# hunt around for good position
# let's do a random-start hillclimber
def randHill(one, two, shape):
# set up empty images
big_empty = np.zeros(shape, np.uint8);
# set global best
g_best_score = -999999;
g_best_pos = None;
# lets do 200 iterations
iters = 200;
for a in range(iters):
# progress check
print(str(a) + " of " + str(iters));
# start with random position
h,w = two.shape[:2];
pos_one = [w,h];
pos_two = randPos(two, shape);
# get score
best_score = fullScore(one, two, pos_one, pos_two, big_empty);
best_pos = [pos_one, pos_two];
# hill climb (only on second image)
while True:
# end condition: no step improves score
end_flag = True;
# 8-way
for y in range(-1, 1+1):
for x in range(-1, 1+1):
if x != 0 or y != 0:
# get score and update
score, new_pos = hillStep(one, two, best_pos, big_empty, [x,y]);
if score > best_score:
best_score = score;
best_pos[1] = new_pos[:];
end_flag = False;
# end
if end_flag:
break;
else:
# show
# prog = overlay(one, big_empty, best_pos[0]);
# prog = overlay(two, prog, best_pos[1]);
# cv2.imshow("prog", prog);
# cv2.waitKey(1);
pass;
# check for new global best
if best_score > g_best_score:
g_best_score = best_score;
g_best_pos = best_pos[:];
print("top score: " + str(g_best_score));
return g_best_score, g_best_pos;
# load both images
top = cv2.imread("top.jpg");
bottom = cv2.imread("bottom.jpg");
top = cv2.cvtColor(top, cv2.COLOR_BGR2GRAY);
bottom = cv2.cvtColor(bottom, cv2.COLOR_BGR2GRAY);
# randomly crop
top = randCrop(top, 20);
bottom = randCrop(bottom, 20);
# randomly add noise
saltPepper(top, 200, 1000);
saltPepper(bottom, 200, 1000);
# set up max image (assume no overlap whatsoever)
tw = 0;
th = 0;
h, w = top.shape;
tw += w;
th += h;
h, w = bottom.shape;
tw += w*2;
th += h*2;
# do random-start hill climb
_, best_pos = randHill(top, bottom, (th, tw));
# show
empty = np.zeros((th, tw), np.uint8);
pos1, pos2 = best_pos;
image = overlay(top, empty, pos1);
image = overlay(bottom, image, pos2);
# do brute force
# small_pos, big_pos, small, big, empty = bruteForce(top, bottom);
# image = overlay(big, empty, big_pos);
# image = overlay(small, image, small_pos);
# recolor overlap
h,w = empty.shape;
color = np.zeros((h,w,3), np.uint8);
l,r,t,b = overlap(top, bottom, pos1, pos2);
color[:,:,0] = image;
color[:,:,1] = image;
color[:,:,2] = image;
color[t:b, l:r, 0] += 100;
# show images
cv2.imshow("top", top);
cv2.imshow("bottom", bottom);
cv2.imshow("overlayed", image);
cv2.imshow("Color", color);
cv2.waitKey(0);
Edit: I added in the random-start hillclimber

How to plot 2D data (image) with x and y coordinates

I would like to plot a 2D spectrum where the x coordinate is the spectral dimension (wavelength) and the y coordinate is the spatial dimension (in arcseconds of the sky) using pyqtgraph.
I've been able to do this using an ImageItem() but I cannot seem to figure out how to display the x and y axes in the right coordinates.
I don't want to just change the labels or the ticks but indeed the coordinates of the plot because I later need to perform operations using these values (wavelength and arcsecs).
Here is a minimal working example:
import pyqtgraph as pg
import numpy as np
# The fake data
wvlg = np.linspace(300, 600, 5000)
arcsec = np.linspace(-5, 5, 100)
flux = np.ones((wvlg.shape[0], arcsec.shape[0])) * np.exp(-(arcsec)**2/0.1)
flux += np.random.normal(0, 0.1, size=(wvlg.shape[0], arcsec.shape[0]))
# The plotting
win = pg.GraphicsLayoutWidget(show=True)
ax2D = win.addPlot(title='2D spectrum', row=0, col=0)
img = pg.ImageItem()
img.setImage(flux)
ax2D.addItem(img)
# Some line converting the x and y values to wvlg and arcsec
This gives an image where the x and y axis show the index value, whereas I would like to show the corresponding wavelength and arcsec values.
Is there an easy way to do this that I just grossly overlooked in the documentation?

You can use the setRect method of the ImageItem class to set the extent of the data. See my example below.
Note that I moved the image by half a pixel so that the pixel centers match the exact coordinates. Otherwise the coordinates would align with one of the pixel's corner points.
import pyqtgraph as pg
import numpy as np
from PyQt5 import QtCore, QtWidgets
def main():
app = QtWidgets.QApplication([])
# The fake data
wvlg = np.linspace(300, 600, 5000)
arcsec = np.linspace(-5, 5, 100)
flux = np.ones((wvlg.shape[0], arcsec.shape[0])) * np.exp(-(arcsec)**2/0.1)
flux += np.random.normal(0, 0.1, size=(wvlg.shape[0], arcsec.shape[0]))
# The plotting
win = pg.GraphicsLayoutWidget()
ax2D = win.addPlot(title='2D spectrum', row=0, col=0)
img = pg.ImageItem()
img.setImage(flux)
ax2D.addItem(img)
print(flux.shape)
# Move the image by half a pixel so that the center of the pixels are
# located at the coordinate values
dx = wvlg[1]-wvlg[0]
dy = arcsec[1]-arcsec[0]
print("pixel size x: {}, pixel size y: {}".format(dx, dy))
rect = QtCore.QRectF(wvlg[0] - dx/2, arcsec[0] - dy/2,
wvlg[-1] - wvlg[0], arcsec[-1] - arcsec[0])
print(rect)
img.setRect(rect)
ax2D.setLabels(left='arcsec', bottom='wvlg')
win.show()
win.raise_()
app.exec_()
if __name__ == "__main__":
main()

OR-Tools Python - AddNoOverlap2D not working - Stock Cutting Problem

Trying to cut given set of rectangles from a large rectangle. The program is running fine but it is not respecting the AddNoOverlap2D constraint.
The program outputs
0, 0 -> 2, 2
0, 0 -> 1, 3
0, 0 -> 4, 3
All the coordinates of rectangles output by program have (0,0) as first point and hence are overlaping. I want get the rectangles that are not overlaping?
I am using model.AddNoOverlap2D constraint and the objective I have set is to minimize the unused area of large rectangle. Complete Code:
from __future__ import print_function
import collections
from ortools.sat.python import cp_model
def StockCutter():
"""Cutting Stock problem."""
# Create the model
model = cp_model.CpModel()
# rect = [width, height]
rects_data = [
[2, 2],
[1, 3],
[4, 3]
]
rect_ids = range(len(rects_data))
# parent rect (to cut from)
horizon = [6, 6]
print("Horizon: ", horizon)
# Named tuple to store information about created variables
rect_type = collections.namedtuple('rect_type', 'x1 y1 x2 y2 x_interval y_interval')
all_vars = {}
# to save area of all small rects, to cut from parent rect
total_area = 0
# x_intervals holds the widths of each rect
x_intervals = collections.defaultdict(list)
# y_intervals holds the lengths of each rect
y_intervals = collections.defaultdict(list)
for rect_id, rect in enumerate(rects_data):
width = rect[0]
height = rect[1]
area = width * height
total_area += area
print(f"Rect: {width}x{height}, Area: {area}")
suffix = '_%i_%i' % (width, height)
# interval to represent width
x1_var = model.NewIntVar(0, horizon[0], 'x1' + suffix)
x2_var = model.NewIntVar(0, horizon[0], 'x2' + suffix)
x_interval_var = model.NewIntervalVar(x1_var, width, x2_var, 'x_interval' + suffix)
# interval to represent height
y1_var = model.NewIntVar(0, horizon[1], 'y1' + suffix)
y2_var = model.NewIntVar(0, horizon[1], 'y2' + suffix)
y_interval_var = model.NewIntervalVar(y1_var, height, y2_var, 'y_interval' + suffix)
all_vars[rect_id] = rect_type(
x1=x1_var,
y1=y1_var,
x2=x2_var,
y2=y2_var,
x_interval=x_interval_var,
y_interval=y_interval_var
)
x_intervals[rect_id].append(x_interval_var)
y_intervals[rect_id].append(y_interval_var)
# NOT WORKING???
for rect_id in rect_ids:
model.AddNoOverlap2D(x_intervals[rect_id], y_intervals[rect_id])
# objective: Area of parent (horizon) is max that the sum of all the rectangles' areas can have
obj_var = model.NewIntVar(0, horizon[0]*horizon[1], 'area')
# minimize the area not used
model.Minimize(obj_var - total_area)
# Solve model
solver = cp_model.CpSolver()
status = solver.Solve(model)
if status == cp_model.OPTIMAL:
# print coords
for rect_id, rect in enumerate(rects_data):
x1=solver.Value(all_vars[rect_id].x1)
y1=solver.Value(all_vars[rect_id].y1)
x2=solver.Value(all_vars[rect_id].x2)
y2=solver.Value(all_vars[rect_id].y2)
print(f"{x1}, {y1} -> {x2}, {y2}")
StockCutter()

You should only call AddNoOverlap2D once with the list of x_intervals and y_intervals:
# x_intervals holds the widths of each rect
x_intervals = []
# y_intervals holds the lengths of each rect
y_intervals = []
for rect_id, rect in enumerate(rects_data):
...
x_intervals.append(x_interval_var)
y_intervals.append(y_interval_var)
model.AddNoOverlap2D(x_intervals, y_intervals)

Surfaceplot Heightcolors

I am doing a animated surface plot via tcp-data.
When the data incoming are small (random 0 - 5) i get a nicely colored Graph, but when i send bigger data (e.g. -50 to +50) the colors get messed up (pictures below), where the whole surface graph is white. I've tried some matplotlib Colormaps, but the result was similiar, with only the white changing to another color, but no surface was visible due to everything having the same color.
Heres my code:
from pyqtgraph.Qt import QtCore, QtGui
import pyqtgraph.opengl as gl
import numpy as np
import datetime
from matplotlib import cm
numberOfData = 1000
widthOfData = 500
x = np.linspace(-widthOfData / 2, widthOfData / 2, widthOfData)
y = np.linspace(-numberOfData / 2, numberOfData / 2, numberOfData)
#colormap = cm.get_cmap('jet') # cm.get_cmap("CMRmap") 'viridis'
#colormap._init()
#lut = (colormap._lut * 255).view(np.ndarray) # Convert matplotlib colormap from 0-1 to 0 -255 for Qt
p4 = gl.GLSurfacePlotItem(x, y, shader='heightColor', computeNormals=False,
smooth=False) # smooth true = faster; dont turn on computenormals
p4.shader()['colorMap'] = np.array([0.2, 2, 0.5, 0.2, 1, 1, 0.2, 0, 2]) #lut
# p4.setGLOptions('opaque')
data = np.zeros((widthOfData, numberOfData), dtype=int)
index = 0
def init():
global p4, data, index
## Create a GL View widget to display data
app = QtGui.QApplication([])
w = gl.GLViewWidget()
w.show()
w.setWindowTitle('PAS Surfaceplot')
w.setGeometry(100, 100, 1500, 800) # distance && resolution
w.setCameraPosition(distance=1000)
## Create axis
# axis = pg.AxisItem('left', pen=None, linkView=None, parent=None, maxTickLength=-5, showValues=True)
# axis.show()
# axis = pg.AxisItem('left', pen = None)
# xAxis.paint()
# Axis.setSize(self.valueNumber, self.valueNumber, self.valueNumber)
# axis.setStyle(showValues = True)
# axis.show()
# --------------------
axis = gl.GLAxisItem()
# xAxis.paint()
# axis.setSize(self.valueNumber, self.valueNumber, self.valueNumber)
w.addItem(axis)
## Add a grid to the view
g = gl.GLGridItem()
g.setSize(x=widthOfData * 2, y=numberOfData * 2)
# g.scale(2,2,1000)
g.setDepthValue(10) # draw grid after surfaces since they may be translucent
w.addItem(g)
## create a surface plot, tell it to use the 'heightColor' shader
## since this does not require normal vectors to render (thus we
## can set computeNormals=False to save time when the mesh updates)
# p4.translate(100, 100, 0)
w.addItem(p4)
# timer = QtCore.QTimer()
# timer.timeout.connect(updateSelf)
# timer.start(20)
## Start Qt event loop unless running in interactive mode.
import sys
if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'):
QtGui.QApplication.instance().exec_()
#update via timer
def updateSelf():
global p4, data, index
timeBeforeUpdate = datetime.datetime.now()
data = np.delete(data, 0, 0)
newValues = np.random.randint(5, size=(1, numberOfData))
# print('newval ', newValues)
data = np.concatenate((data, newValues))
p4.setData(z=data)
timeAfterUpdate = datetime.datetime.now()
timeDiff = timeAfterUpdate - timeBeforeUpdate
elapsed_ms = (timeDiff.days * 86400000) + (timeDiff.seconds * 1000) + (timeDiff.microseconds / 1000)
# print(elapsed_ms, ' ms')
#update via tcp
def update(framesList):
global p4, data, index
timeBeforeUpdate = datetime.datetime.now()
for frame in framesList:
data = np.delete(data, 0, 0)
frame = np.array(frame, ndmin=2)
# print('data: ', data)
# print('frame: ', frame)
data = np.concatenate((data, frame))
p4.setData(z=data)
timeAfterUpdate = datetime.datetime.now()
timeDiff = timeAfterUpdate - timeBeforeUpdate
elapsed_ms = (timeDiff.days * 86400000) + (timeDiff.seconds * 1000) + (timeDiff.microseconds / 1000)
print(elapsed_ms, ' ms')
# init()
# timer = QtCore.QTimer()
# timer.timeout.connect(updateSelf)
# timer.start(20)
How do i fix this?
working colors
messed up colors

i have solved the issue with
self.surfacePlot.shader()['colorMap'] = np.array([0.01, 40, 0.5, 0.01, 40, 1, 0.01, 40, 2]) # lut
The problem was indeed that the color brightens way too fast.
Colormaps are defined by 3 triples, where index_0^index_3 determines the color. So in my case it is now 0.01^0.5, 0.01^1 and 0.01^2.
Sadly i can't find the wiki-code example where i found this in a comment

From what I can think of, the unified color is because of high density of the data. Since the variation occurs as you increase your data, what you can try is reducing the range of data, i.e. for -50 to 50, try dividing it first by 10, so as to get the values in range of -10 to 10. That should help.

Kinect V2 depth image produces strange point cloud coordinates when using getPointXYZ() from pylibfreenect2?

I am trying to generate the real-world coordinates from my MS Kinect V2.
I have managed to piece together a pyqt + opengl scatter plot and show the depth data from the Kinect using pylibfreenect2.
I noticed immediately that the depth data was not the same as point cloud data. Notice my room's ceiling is very distorted (what should be a flat ceiling begins to resemble a hockey stick graph)
Result of plotting the depth frame
After some reading and digging through source files I managed to find a function which seemed very promising.
getPointXYZ - Construct a 3-D point in a point cloud.
As it only works on one pixel at a time I wrote a simple nested for loop. In the code below you should see the lines:
out = np.zeros((d.shape[0]*d.shape[1], 3)) #shape = (217088, 3)
for row in range(d.shape[0]):
for col in range(d.shape[1]):
world = registration.getPointXYZ(undistorted, row, col) #convert depth pixel to real-world coordinate
out[row + col] = world
Result of coordinates from getPointXYZ()
Not sure what's going on there. It looks more like a straight line and sometimes its resembles a rectangle and it's very flat (yet it sits at arbitrary angels in all three dimensions). When I move my hand in front of the sensor I can see some points move around but no declarable shapes are visible. It appears that all points are being crammed together.
The following is a Python script that will show a pyQt application window containing an openGL scatter plot. Frames are received from the Kinect sensor through pylibfreenect2 and the scatter plot's points are generated by iterating over each row and column of the depth data and sending it through getPointXYZ (This is really slow and doesn't work...).
# coding: utf-8
# An example using startStreams
from pyqtgraph.Qt import QtCore, QtGui
import pyqtgraph.opengl as gl
import numpy as np
import cv2
import sys
from pylibfreenect2 import Freenect2, SyncMultiFrameListener
from pylibfreenect2 import FrameType, Registration, Frame, libfreenect2
fn = Freenect2()
num_devices = fn.enumerateDevices()
if num_devices == 0:
print("No device connected!")
sys.exit(1)
serial = fn.getDeviceSerialNumber(0)
device = fn.openDevice(serial)
types = 0
types |= FrameType.Color
types |= (FrameType.Ir | FrameType.Depth)
listener = SyncMultiFrameListener(types)
# Register listeners
device.setColorFrameListener(listener)
device.setIrAndDepthFrameListener(listener)
device.start()
# NOTE: must be called after device.start()
registration = Registration(device.getIrCameraParams(),
device.getColorCameraParams())
undistorted = Frame(512, 424, 4)
registered = Frame(512, 424, 4)
#QT app
app = QtGui.QApplication([])
w = gl.GLViewWidget()
w.show()
g = gl.GLGridItem()
w.addItem(g)
#initialize some points data
pos = np.zeros((1,3))
sp2 = gl.GLScatterPlotItem(pos=pos)
w.addItem(sp2)
def update():
frames = listener.waitForNewFrame()
ir = frames["ir"]
color = frames["color"]
depth = frames["depth"]
d = depth.asarray()
registration.apply(color, depth, undistorted, registered)
#There are 3 optionally commented methods for generating points data (the last one is not commented here).
#First will generate points using depth data only.
#Second will generate colored points and pointcloud xyz coordinates.
#Third is simply the pointcloud xyz coordinates without the color mapping.
"""
#Format depth data to be displayed
m, n = d.shape
R, C = np.mgrid[:m, :n]
out = np.column_stack((d.ravel() / 4500, C.ravel()/m, (-R.ravel()/n)+1))
"""
"""
#Format undistorted and regisered data to real-world coordinates with mapped colors (dont forget color=out_col in setData)
out = np.zeros((d.shape[0]*d.shape[1], 3)) #shape = (217088, 3)
out_col = np.zeros((d.shape[0]*d.shape[1], 3)) #shape = (217088, 3)
for row in range(d.shape[0]):
for col in range(d.shape[1]):
world = registration.getPointXYZRGB(undistorted, registered, row, col)
out[row + col] = world[0:3]
out_col[row + col] = np.array(world[3:6]) / 255
"""
# Format undistorted data to real-world coordinates
out = np.zeros((d.shape[0]*d.shape[1], 3)) #shape = (217088, 3)
for row in range(d.shape[0]):
for col in range(d.shape[1]):
world = registration.getPointXYZ(undistorted, row, col)
out[row + col] = world
sp2.setData(pos=out, size=2)
listener.release(frames)
t = QtCore.QTimer()
t.timeout.connect(update)
t.start(50)
## Start Qt event loop unless running in interactive mode.
if __name__ == '__main__':
import sys
if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'):
QtGui.QApplication.instance().exec_()
device.stop()
device.close()
sys.exit(0)
I am unsure what I should do next in order to get the actual point cloud coordinate data.
Does anyone have any suggestions as to what I'm doing wrong?
My operating system is Ubuntu 16.0.4 with Python 3.5
Thanks.

The answer was actually to resolve a mistake I made in those nested loops. I noticed it was not indexing an array correctly:
#From:
out[row + col]
#To:
out[row * n_columns + col]
Vertexes are now accurately positioned in 3d space and all looks good!
Here's the revised and fully functional code:
# coding: utf-8
# An example using startStreams
from pyqtgraph.Qt import QtCore, QtGui
import pyqtgraph.opengl as gl
import numpy as np
import cv2
import sys
from pylibfreenect2 import Freenect2, SyncMultiFrameListener
from pylibfreenect2 import FrameType, Registration, Frame, libfreenect2
fn = Freenect2()
num_devices = fn.enumerateDevices()
if num_devices == 0:
print("No device connected!")
sys.exit(1)
serial = fn.getDeviceSerialNumber(0)
device = fn.openDevice(serial)
types = 0
types |= FrameType.Color
types |= (FrameType.Ir | FrameType.Depth)
listener = SyncMultiFrameListener(types)
# Register listeners
device.setColorFrameListener(listener)
device.setIrAndDepthFrameListener(listener)
device.start()
# NOTE: must be called after device.start()
registration = Registration(device.getIrCameraParams(),
device.getColorCameraParams())
undistorted = Frame(512, 424, 4)
registered = Frame(512, 424, 4)
#QT app
app = QtGui.QApplication([])
w = gl.GLViewWidget()
w.show()
g = gl.GLGridItem()
w.addItem(g)
#initialize some points data
pos = np.zeros((1,3))
sp2 = gl.GLScatterPlotItem(pos=pos)
w.addItem(sp2)
def update():
colors = ((1.0, 1.0, 1.0, 1.0))
frames = listener.waitForNewFrame()
ir = frames["ir"]
color = frames["color"]
depth = frames["depth"]
d = depth.asarray()
registration.apply(color, depth, undistorted, registered)
listener.release(frames)
"""
#Format raw depth data to be displayed
m, n = d.shape
R, C = np.mgrid[:m, :n]
out = np.column_stack((d.ravel() / 4500, C.ravel()/m, (-R.ravel()/n)+1))
"""
#Format undistorted and regisered data to real-world coordinates with mapped colors (dont forget color=out_col in setData)
n_rows = d.shape[0]
n_columns = d.shape[1]
out = np.zeros((d.shape[0] * d.shape[1], 3), dtype=np.float64)
colors = np.zeros((d.shape[0] * d.shape[1], 3), dtype=np.float64)
for row in range(n_rows):
for col in range(n_columns):
X, Y, Z, B, G, R = registration.getPointXYZRGB(undistorted, registered, row, col)
out[row * n_columns + col] = np.array([X, Y, Z]) # np.array(pt, dtype=np.float64)
colors[row * n_columns + col] = np.divide([R, G, B], 255) # np.array(pt, dtype=np.float64)
"""
#Format undistorted depth data to real-world coordinates
n_rows = d.shape[0]
n_columns = d.shape[1]
out = np.zeros((d.shape[0] * d.shape[1], 3), dtype=np.float64)
for row in range(n_rows):
for col in range(n_columns):
X, Y, Z = registration.getPointXYZ(undistorted, row, col)
out[row * n_columns + col] = np.array([X, Y, Z]) # np.array(pt, dtype=np.float64)
"""
sp2.setData(pos=np.array(out, dtype=np.float64), color=colors, size=2)
t = QtCore.QTimer()
t.timeout.connect(update)
t.start(50)
## Start Qt event loop unless running in interactive mode.
if __name__ == '__main__':
import sys
if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'):
QtGui.QApplication.instance().exec_()
device.stop()
device.close()
sys.exit(0)
[EDIT]
Please see This Post for additional information