I'm working on some optimazation and want to convert some parts from python to c++
Is it possible to convert this code to c++ with opencv?
The python code uses numpy
import numpy as np
from PIL import Image
pil_img = Image.open(input_filename)
img = np.array(pil_img)
pixels = img.reshape((-1, 3))
num_pixels = pixels.shape[0]
num_samples = int(num_pixels*5)
idx = np.arange(num_pixels)
np.random.shuffle(idx)
samples = pixels[idx[:num_samples]]
update
std::vector<uchar> sample_pixels(const cv::Mat& m, int sample_percent=5){
assert(m.isContinuous());
const auto* input = m.ptr<uchar>();
int
num_pixels = m.total(),
num_samples = num_pixels * sample_percent;
std::cout
<< "num pixels: " << num_pixels << '\n'
<< "num samples: " << num_samples << '\n';
std::vector<uchar> samples(num_samples);
// Fills idx with sequentially increasing values
std::vector<int> idx(num_pixels);
std::iota(idx.begin(), idx.end(), 0);
// Shuffle idx
std::mt19937 engine(0);
std::shuffle(idx.begin(), idx.end(), engine);
for(int i = 0; i < num_samples; i++){
//samples[i] = input[idx[i]];
}
//auto output_mat = cv::Mat(samples, false);
//cv::imwrite("enhance-samples.png", output_mat);
return samples;
}
This is the equivalent code in C++11. This should be several times faster than your python code.
#include <random>
#include <numeric>
#include <opencv2/opencv.hpp>
void shuffling(const std::string &input_filename, const std::string &output_filename) {
// ========== UPDATE ==========
const cv::Mat plain_input_mat = cv::imread(input_filename, -1);
// Equivalent to img.reshape((-1, 3))
const cv::Mat input_mat = plain_input_mat.reshape(3);
// ============================
// By doing this, you can access the pixels without any extra checks.
assert(input_mat.isContinuous());
const auto *input = input_mat.ptr<cv::Vec3b>();
const auto num_samples = input_mat.total();
std::vector<cv::Vec3b> output(num_samples);
std::vector<int> idx(input_mat.total());
std::iota(idx.begin(), idx.end(), 0); // Equivalent to arange.
// Note: numpy uses PCG64 which does not exist in the std library.
std::mt19937 engine(0);
std::shuffle(idx.begin(), idx.end(), engine);
for (int i = 0; i < num_samples; i++) {
output[i] = input[idx[i]];
}
// Save as an image if necessary.
auto output_mat = cv::Mat(output, false);
cv::imwrite(output_filename, output_mat);
}
There are a couple of additional notes.
Note1: Due to the difference in the shuffle algorithm between python and std, the results are not exactly the same.
Note2: With your code, num_samples cannot be larger than the number of pixels in the input image, which seems to be a bug. Please check the length of the samples.
Note3: In both implementations, the most expensive part is shuffle. 60% for python and more than 80% for C++ is spent here. If you want to optimize further, this is definitely where you should exploit.
Does anyone know how I can process a triangle mesh to allow conversion into a HalfEdgeTriangleMesh. I am trying to use the get_boundaries in HalfEdge to detect edges, which is not an included function in TriangleMesh
The last line gives the error.
mesh = mesh.remove_duplicated_triangles()
mesh = mesh.remove_degenerate_triangles()
mesh = mesh.remove_duplicated_vertices()
mesh = mesh.remove_non_manifold_edges()
mesh = mesh.remove_unreferenced_vertices()
half_edge_mesh = o3d.geometry.HalfEdgeTriangleMesh.create_from_triangle_mesh(mesh)
In Open3d,if you want to create HalfEdge from triangle mesh,please make sure your mesh is manifold and without singular vertex.In the case of a vertex has more than one output halfedge,runtime error will occur.In the newest version of Open3d,this bug has not been repaired.Open3d Link
for (size_t triangle_index = 0;
triangle_index < mesh_cpy->triangles_.size(); triangle_index++) {
const Eigen::Vector3i &triangle = mesh_cpy->triangles_[triangle_index];
size_t num_half_edges = het_mesh->half_edges_.size();
size_t he_0_index = num_half_edges;
size_t he_1_index = num_half_edges + 1;
size_t he_2_index = num_half_edges + 2;
HalfEdge he_0(Eigen::Vector2i(triangle(0), triangle(1)),
int(triangle_index), int(he_1_index), -1);
HalfEdge he_1(Eigen::Vector2i(triangle(1), triangle(2)),
int(triangle_index), int(he_2_index), -1);
HalfEdge he_2(Eigen::Vector2i(triangle(2), triangle(0)),
int(triangle_index), int(he_0_index), -1);
if (vertex_indices_to_half_edge_index.find(he_0.vertex_indices_) !=
vertex_indices_to_half_edge_index.end() ||
vertex_indices_to_half_edge_index.find(he_1.vertex_indices_) !=
vertex_indices_to_half_edge_index.end() ||
vertex_indices_to_half_edge_index.find(he_2.vertex_indices_) !=
vertex_indices_to_half_edge_index.end()) {
utility::LogError(
"ComputeHalfEdges failed. Duplicated half-edges.");//Throw a runtime error.
}
het_mesh->half_edges_.push_back(he_0);
het_mesh->half_edges_.push_back(he_1);
het_mesh->half_edges_.push_back(he_2);
vertex_indices_to_half_edge_index[he_0.vertex_indices_] = he_0_index;
vertex_indices_to_half_edge_index[he_1.vertex_indices_] = he_1_index;
vertex_indices_to_half_edge_index[he_2.vertex_indices_] = he_2_index;
}
singular vertices
just like red arrows vertices.In these vertices,error will occur.
If you want to find boundary,please try Openmesh or Libigl.Both Openmesh and Libigl have python version.
I have a model in caffe that produce a multi-dimensional array. Size of this array is [1x10x8x8] so in python I haven't problem with this size because python automatically manage this array and I know order of elements in that. but when I switch to opencv with c++ the whole array is a vector and I haven't any idea how to regenerate something like python array, I use cv::NAryMatIterator to access multi-dimensional array such below
const cv::Mat* arrays[]={&prob,0}; //my multi-dimensional array is prob
cv::Mat my_planes[1];
cv::NAryMatIterator it(arrays,my_planes);
cv::Mat Multi_Array ; //temporary Mat
for (int p = 0; p < it.nplanes; ++p,++it) {
Multi_Array = it.planes[0];
}
after doing that I see Multi_Array size is [640x1] which seems that it is equal to 8x8x10 that python produced. is there anyway to access 8x8 planes one by one?
EDIT: my multi-dimensional array size is [1x10x8x8]
To access the 3D array as if it were a 2D array with shape [640][1], you could write 3 loops to iterate on the elements using a [x,y,z] format like:
int data[640][1] = { 0 };
int width = 8, height = 8, depth = 10;
for (int x = 0; x < width; x++)
for (int y = 0; y < height; y++)
for (int z = 0; z < depth; z++)
{
int idx = x * height * depth + y * depth + z;
data[idx][0] = idx;
}
This fills the array with numbers ranging from 0 to 639.
If you are looking to access a 2D array as a 1D, check this answer.
If your model data is ordered in row-major form, you can have OpenCV interpret the data as a Mat of the required size. Then, planes of the Mat can be accessed using multidim_mat.row( row_number ).
In order to create a Mat from the data:
int data[640] = { 0 };
const int size[] = { 8, 8, 10 };
cv::Mat multidim_mat(3, size, CV_32S, data);
std::cout << multidim_mat.dims << std::endl;
for (int i = 0; i < multidim_mat.dims; i++) {
std::cout << "Dimension " << i << " is of size " << multidim_mat.size[i] << std::endl;
}
The CV_32S is to inform OpenCV to interpret the data as signed 32-bit integers.
References: https://docs.opencv.org/3.4.0/d3/d63/classcv_1_1Mat.html#a5fafc033e089143062fd31015b5d0f40, https://docs.opencv.org/3.4.0/d3/d63/classcv_1_1Mat.html#details,
In first step we need to get a pointer to OpenCV Mat object, you can do this by the below command.(I assume that data that represent your data is primarily float and consider probability Mat is prob which we get this Mat from caffe)
float* p = (float*)(prob.data);
This pointer will points to the where data is reside in memory. So for example if we want to get access to the element in (1,3,7,7) location we can do this operation like this:
int S= sizeof(float);
float val = p[(
7*p.step[3]/S + //forth dimension
7*p.step[2]/S + //third dimension
3*p.step[1]/S //second dimension
)]
//first dimension is not needed, because it is decoded in address of p
//and if you have any higher number than 1 in first dimension you need to add it to the above command
So for traversing in the probability matrix you can do that like the below:
auto S=sizeof(float);
for (int d2 = 0; d2 < 129; ++d2) {
for (int d3 = 0; d3 < 129; ++d3) {
for (int d4 = 0; d4 < 10; ++d4) {
float val = p[(d2*prob.step[3]/S + d3*prob.step[2]/S + d4* prob.step[1]/S)];
}
}
}
Based on a disparity matrix from a passive stereo-camera system i need to calculate a v-disparity representation for obstacle detection with OpenCV.
A working implementation is not the problem. The problem is to do it fast...
(One) Reference for v-Disparity: Labayrade, R. and Aubert, D. and Tarel, J.P.
Real time obstacle detection in stereovision on non flat road geometry through v-disparity representation
The basic in short, to get the v-disparity (figure 1), is to analyze the rows of the disparity-matrix (figure 2) an represent the result as a histogram for each row over the disparity values. u-disparity (figure 3) is the same on the columns of the disparity-matrix. (All figures are false-colored.)
I have implement the "same" in Python and C++. The speed in Python is acceptable but in C++ i get for the u- and v-disparity a time round about a half second (0.5 s).
(1. edit: due to the separate time measurement, only the calculation of the u-histogram takes a big amount of time...)
This leads me to following questions:
Is it possible to avoid the loops for the line-wise calculation of the histogram? Is there a "trick" to do it with one call of calcHist-Function from OpenCV? Perhaps with the dimensions?
Is it in C++ just bad-coded and the runtime-issue are not related to the loops used for calculation?
Thanks, all
Working implementation in Python:
#!/usr/bin/env python2
#-*- coding: utf-8 -*-
#
# THIS SOURCE-CODE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED. IN NO EVENT WILL THE AUTHOR BE HELD LIABLE FOR ANY DAMAGES ARISING FROM
# THE USE OF THIS SOURCE-CODE. USE AT YOUR OWN RISK.
import cv2
import numpy as np
import time
def draw_object(image, x, y, width=50, height=100):
color = image[y, x]
image[y-height:y, x-width//2:x+width//2] = color
IMAGE_HEIGHT = 600
IMAGE_WIDTH = 800
while True:
max_disp = 200
# create fake disparity
image = np.zeros((IMAGE_HEIGHT, IMAGE_WIDTH), np.uint8)
for c in range(IMAGE_HEIGHT)[::-1]:
image[c, ...] = int(float(c) / IMAGE_HEIGHT * max_disp)
draw_object(image, 275, 175)
draw_object(image, 300, 200)
draw_object(image, 100, 350)
# calculate v-disparity
vhist_vis = np.zeros((IMAGE_HEIGHT, max_disp), np.float)
for i in range(IMAGE_HEIGHT):
vhist_vis[i, ...] = cv2.calcHist(images=[image[i, ...]], channels=[0], mask=None, histSize=[max_disp],
ranges=[0, max_disp]).flatten() / float(IMAGE_HEIGHT)
vhist_vis = np.array(vhist_vis * 255, np.uint8)
vblack_mask = vhist_vis < 5
vhist_vis = cv2.applyColorMap(vhist_vis, cv2.COLORMAP_JET)
vhist_vis[vblack_mask] = 0
# calculate u-disparity
uhist_vis = np.zeros((max_disp, IMAGE_WIDTH), np.float)
for i in range(IMAGE_WIDTH):
uhist_vis[..., i] = cv2.calcHist(images=[image[..., i]], channels=[0], mask=None, histSize=[max_disp],
ranges=[0, max_disp]).flatten() / float(IMAGE_WIDTH)
uhist_vis = np.array(uhist_vis * 255, np.uint8)
ublack_mask = uhist_vis < 5
uhist_vis = cv2.applyColorMap(uhist_vis, cv2.COLORMAP_JET)
uhist_vis[ublack_mask] = 0
image = cv2.applyColorMap(image, cv2.COLORMAP_JET)
cv2.imshow('image', image)
cv2.imshow('vhist_vis', vhist_vis)
cv2.imshow('uhist_vis', uhist_vis)
cv2.imwrite('disparity_image.png', image)
cv2.imwrite('v-disparity.png', vhist_vis)
cv2.imwrite('u-disparity.png', uhist_vis)
if chr(cv2.waitKey(0)&255) == 'q':
break
Working implementation in C++:
#include <iostream>
#include <stdlib.h>
#include <ctime>
#include <opencv2/opencv.hpp>
using namespace std;
void draw_object(cv::Mat image, unsigned int x, unsigned int y, unsigned int width=50, unsigned int height=100)
{
image(cv::Range(y-height, y), cv::Range(x-width/2, x+width/2)) = image.at<unsigned char>(y, x);
}
int main()
{
unsigned int IMAGE_HEIGHT = 600;
unsigned int IMAGE_WIDTH = 800;
unsigned int MAX_DISP = 250;
unsigned int CYCLE = 0;
//setenv("QT_GRAPHICSSYSTEM", "native", 1);
// === PREPERATIONS ==
cv::Mat image = cv::Mat::zeros(IMAGE_HEIGHT, IMAGE_WIDTH, CV_8U);
cv::Mat uhist = cv::Mat::zeros(IMAGE_HEIGHT, MAX_DISP, CV_32F);
cv::Mat vhist = cv::Mat::zeros(MAX_DISP, IMAGE_WIDTH, CV_32F);
cv::Mat tmpImageMat, tmpHistMat;
float value_ranges[] = {(float)0, (float)MAX_DISP};
const float* hist_ranges[] = {value_ranges};
int channels[] = {0};
int histSize[] = {MAX_DISP};
struct timespec start, finish;
double elapsed;
while(1)
{
CYCLE++;
// === CLEANUP ==
image = cv::Mat::zeros(IMAGE_HEIGHT, IMAGE_WIDTH, CV_8U);
uhist = cv::Mat::zeros(IMAGE_HEIGHT, MAX_DISP, CV_32F);
vhist = cv::Mat::zeros(MAX_DISP, IMAGE_WIDTH, CV_32F);
// === CREATE FAKE DISPARITY WITH OBJECTS ===
for(int i = 0; i < IMAGE_HEIGHT; i++)
image.row(i) = ((float)i / IMAGE_HEIGHT * MAX_DISP);
draw_object(image, 200, 500);
draw_object(image, 525 + CYCLE%100, 275);
draw_object(image, 500, 300 + CYCLE%100);
clock_gettime(CLOCK_MONOTONIC, &start);
// === CALCULATE V-HIST ===
for(int i = 0; i < IMAGE_HEIGHT; i++)
{
tmpImageMat = image.row(i);
vhist.row(i).copyTo(tmpHistMat);
cv::calcHist(&tmpImageMat, 1, channels, cv::Mat(), tmpHistMat, 1, histSize, hist_ranges, true, false);
vhist.row(i) = tmpHistMat.t() / (float) IMAGE_HEIGHT;
}
clock_gettime(CLOCK_MONOTONIC, &finish);
elapsed = (finish.tv_sec - start.tv_sec);
elapsed += (finish.tv_nsec - start.tv_nsec) * 1e-9;
cout << "V-HIST-TIME: " << elapsed << endl;
clock_gettime(CLOCK_MONOTONIC, &start);
// === CALCULATE U-HIST ===
for(int i = 0; i < IMAGE_WIDTH; i++)
{
tmpImageMat = image.col(i);
uhist.col(i).copyTo(tmpHistMat);
cv::calcHist(&tmpImageMat, 1, channels, cv::Mat(), tmpHistMat, 1, histSize, hist_ranges, true, false);
uhist.col(i) = tmpHistMat / (float) IMAGE_WIDTH;
}
clock_gettime(CLOCK_MONOTONIC, &finish);
elapsed = (finish.tv_sec - start.tv_sec);
elapsed += (finish.tv_nsec - start.tv_nsec) * 1e-9;
cout << "U-HIST-TIME: " << elapsed << endl;
// === PREPARE AND SHOW RESULTS ===
uhist.convertTo(uhist, CV_8U, 255);
cv::applyColorMap(uhist, uhist, cv::COLORMAP_JET);
vhist.convertTo(vhist, CV_8U, 255);
cv::applyColorMap(vhist, vhist, cv::COLORMAP_JET);
cv::imshow("image", image);
cv::imshow("uhist", uhist);
cv::imshow("vhist", vhist);
if ((cv::waitKey(1)&255) == 'q')
break;
}
return 0;
}
Figure 1: v-disparity
Figure 2: Fake disparity matrix
Figure 3: u-disparity
edit:
correct name for u- and v-disparity and separate time measurement in c++ example
small typo
Today i had the possibility to reinvestigate the problem. Remembering the OpenCV basics (1) for the Mat-structure and the fact that only one calculation takes a huge amount of time, i had the solution.
In OpenCV, each row of an image could be reached by a row-pointer. For iterating columns (done in u-disparity calculation) i suspect, that OpenCV needs to resolve every row-pointer + column-offset for building the histogram.
Changing the Code in a way, that OpenCV is able to use row-pointer, solves the problem for me.
| old code [s] | changed [s]
------------+--------------+-------------
V-HIST-TIME | 0.00351909 | 0.00334152
U-HIST-TIME | 0.600039 | 0.00449285
So for the u-hist-loop i transpose the image and reverse the operation after the loop. The line wise access for calculation could now be done via the row-pointer.
Changed Codelines:
// === CALCULATE U-HIST ===
image = image.t();
for(int i = 0; i < IMAGE_WIDTH; i++)
{
tmpImageMat = image.row(i);
uhist.col(i).copyTo(tmpHistMat);
cv::calcHist(&tmpImageMat, 1, channels, cv::Mat(), tmpHistMat, 1, histSize, hist_ranges, true, false);
uhist.col(i) = tmpHistMat / (float) IMAGE_WIDTH;
}
image = image.t();
Finally my second question takes effect, the runtime-issue belongs not to the loop. A time less than 5 ms is (for now) fast enough.
Very nice code and very illustrative. It helped me understand u-disparity. However, your C/C++ code is broken. I fixed him with this code:
cv::Mat uhist = cv::Mat::zeros(MAX_DISP, IMAGE_WIDTH, CV_32F);
cv::Mat vhist = cv::Mat::zeros(IMAGE_WIDTH, MAX_DISP, CV_32F);
I currently have two matching point sets built into a numpy array of float32:
points1 =
[[ 346.70220947 9076.38476562]
[ 922.99554443 9096.4921875 ]
[ 776.96466064 9108.79101562]
[ 449.0173645 9080.61816406]
[ 2843.19433594 1226.93212891]
[ 779.95275879 9094.76855469]
[ 451.46853638 9092.5078125 ]
[ 3981.4621582 1237.50964355]
[ 132.38700867 9086.7890625 ]
[ 819.10943604 8286.74023438]
[ 1963.64025879 1220.06921387]
[ 1253.79321289 9095.75292969]]
points2 =
[[ 55110.36328125 9405.07519531]
[ 55686.71875 9423.63574219]
[ 55540.8515625 9435.80078125]
[ 55212.58203125 9408.00585938]
[ 57598.76171875 1551.92956543]
[ 55543.78125 9421.88769531]
[ 55214.40625 9420.46972656]
[ 58737.41796875 1561.14831543]
[ 54895.9296875 9414.58203125]
[ 55581.87109375 8613.87011719]
[ 56718.76953125 1546.02197266]
[ 56017.8125 9422.52050781]]
and I'm trying to run:
affine = cv2.estimateRigidTransform(points2,points1,True)
print affine
so that I can generate an affine matrix that can then be translated into a world file (.tfw). The world file is for GIS software that will project these on-the-fly.
At the moment I am getting an error:
Both input images must have either 8uC1 or 8uC3 type in function cvEstimateRigidTransform
I'm not really sure what's going on here. I thought I could use two points sets as parameters as long as I have 6 or more pairs.
Any thoughts or recommendations would be much appreciated!
I had the same weird error but in Java. In my case, It seemed that estimateRigidTransform couldn't recognize that the two Mat images I was giving where actually 2D Point Sets. So I applied a workaround in order to convert my match points from MatOfKeyPoint to MatOfPoint2f type.
Here is the complete Java code (It's not Python, but maybe it will help you):
UPDATE: Filtering your matches is important, cause if you don't you may get an empty array as a result of the transform.
FeatureDetector detector = FeatureDetector.create(FeatureDetector.ORB);
DescriptorExtractor descriptor = DescriptorExtractor.create(DescriptorExtractor.ORB);
DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_HAMMING);
// Load First Image
Mat img1 = Imgcodecs.imread("img1_path", Imgcodecs.IMREAD_GRAYSCALE);
Mat img1_descriptors = new Mat();
MatOfKeyPoint img1_keypoints_mat = new MatOfKeyPoint();
// Detect KeyPoints
detector.detect(img1, img1_keypoints_mat);
descriptor.compute(img1, img1_keypoints_mat, img1_descriptors);
// Load Second Image
Mat img2 = Imgcodecs.imread("img2_path", Imgcodecs.IMREAD_GRAYSCALE);
Mat img2_descriptors = new Mat();
MatOfKeyPoint img2_keypoints_mat = new MatOfKeyPoint();
// Detect KeyPoints
detector.detect(img2, img2_keypoints_mat);
descriptor.compute(img2, img2_keypoints_mat, img2_descriptors);
// Match KeyPoints
MatOfDMatch matOfDMatch = new MatOfDMatch();
matcher.match(img1_descriptors, img2_descriptors, matOfDMatch);
// Filtering the matches
List<DMatch> dMatchList = matOfDMatch.toList();
Double max_dist = 0.0;
Double min_dist = 100.0;
for(int i = 0; i < img1_descriptors.rows(); i++){
Double dist = (double) dMatchList.get(i).distance;
if(dist < min_dist) min_dist = dist;
if(dist > max_dist) max_dist = dist;
}
LinkedList<DMatch> good_matches = new LinkedList<>();
for(int i = 0; i < img1_descriptors.rows(); i++){
if(dMatchList.get(i).distance < 3*min_dist){
good_matches.addLast(dMatchList.get(i));
}
}
// Converting to MatOfPoint2f format
LinkedList<Point> img1_points_list = new LinkedList<>();
LinkedList<Point> img2_points_list = new LinkedList<>();
List<KeyPoint> img1_keyPoints_list = img1_keypoints_mat.toList();
List<KeyPoint> img2_keyPoints_list = img2_keypoints_mat.toList();
int limit = good_matches.size();
for(int i = 0; i < limit; i++){
img1_points_list.addLast(img1_keyPoints_list.get(good_matches.get(i).queryIdx).pt);
img2_points_list.addLast(img2_keyPoints_list.get(good_matches.get(i).trainIdx).pt);
}
MatOfPoint2f img1_point2f_mat = new MatOfPoint2f();
img1_point2f_mat.fromList(img1_points_list);
MatOfPoint2f img2_point2f_mat = new MatOfPoint2f();
img2_point2f_mat.fromList(img2_points_list);
// Draw match points
Mat output = new Mat();
Features2d.drawMatches(img1, img1_keypoints_mat, img2, img2_keypoints_mat, matOfDMatch, output);
Imgcodecs.imwrite("output.png", output);
Mat result = Video.estimateRigidTransform(img1_point2f_mat, img2_point2f_mat, true);
printMat(result); // Printing the optimal affine transformation 2x3 array
// The following variables correspond to the estimateRigidTransform result as shown here: https://stackoverflow.com/a/29511091/5165833
double a = result.get(0,0)[0];
double b = result.get(0,1)[0];
double d = result.get(1,1)[0];
double c = result.get(1,0)[0];
// Solving for scale as shown in the link above
double scale_x = Math.signum(a) * Math.sqrt( (a*a) + (b*b) );
double scale_y = Math.signum(d) * Math.sqrt( (c*c) + (d*d) );
System.out.println("a = "+a);
System.out.println("b = "+b);
System.out.println("scale_x = "+scale_x);
System.out.println("scale_y = "+scale_y);
}
public static void printMat(Mat m)
{
for (int x=0; x < m.height(); x++)
{
for (int y=0; y < m.width(); y++)
{
System.out.printf("%f",m.get(x,y)[0]);
System.out.printf("%s"," ");
}
System.out.printf("\n");
}
}