Question
How can I optimize the process to find the best 4-sided shape that contains my mask?
I am currently working on a project where I need to detect pool table and pool balls from a video frame.
I then need to recreate the state of the game in a 2D mini-map, and in order to do so I need the edges of the pool table's playing field. I have been trying to obtain them from the mask found in the previous step but I get quite unsatisfactory results. How could I improve them?
Code for the detection of the playing field:
Mat src = imread(src_path);
assert(!src.empty());
imshow("Source", src);
Mat hsv;
cvtColor(src, hsv, COLOR_BGR2HSV);
imshow("HSV", hsv);
/* * * * * * * * * * * * * * * * * * * * *
* Start processing BG and PF Masks *
* * * * * * * * * * * * * * * * * * * * */
Mat sharp;
Mat sharpening_kernel = (Mat_<double>(3, 3) << -1, -1, -1,
-1, 9, -1,
-1, -1, -1);
filter2D(hsv, sharp, -1, sharpening_kernel);
//imshow("Sharpening", sharp);
double sigma=1;
GaussianBlur(sharp, sharp, Size(), sigma);
//imshow("Blurred Sharp", sharp);
//waitKey(0);
int min_region_area = int(min_region_area_factor * sharp.cols * sharp.rows); // small region will be ignored
int max_region_area = int(max_region_area_factor * sharp.cols * sharp.rows); // same for too big ones
// "dest" records all regions using different padding number
// 0 - undetermined, 255 - ignored, other number - determined
uchar padding = 1; // use which number to pad in "dest"
Mat dest = Mat::zeros(sharp.rows, sharp.cols, CV_8UC1);
// "mask" records current region, always use "1" for padding
Mat mask = Mat::zeros(sharp.rows, sharp.cols, CV_8UC1);
//Mask to save
Mat final_mask;
// traversal the whole image, apply "seed grow" in undetermined pixels
for (int x=0; x<sharp.cols; ++x) {
for (int y=0; y<sharp.rows; ++y) {
if (dest.at<uchar>(Point(x, y)) == 0) {
grow(sharp, dest, mask, Point(x, y), thresh);
int mask_area = (int)sum(mask).val[0]; // calculate area of the region that we get in "seed grow"
if (mask_area > min_region_area && mask_area < max_region_area) {
dest = dest + mask * padding; // record new region to "dest"
final_mask = mask*255;
//waitKey();
if(++padding > max_region_num) {
printf("run out of max_region_num...");
return -1;
}
} else {
dest = dest + mask * 255; // record as "ignored"
}
mask = mask - mask; // zero mask, ready for next "seed grow"
}
}
}
The grow function is as follows:
void grow(Mat& src, Mat& dest, Mat& mask, Point seed, int thresh) { //thresh=200
/* apply "seed grow" in a given seed
* Params:
* src: source image
* dest: a matrix records which pixels are determined/undtermined/ignored
* mask: a matrix records the region found in current "seed grow"
*/
stack<Point> point_stack;
point_stack.push(seed);
while(!point_stack.empty()) {
Point center = point_stack.top();
mask.at<uchar>(center) = 1;
point_stack.pop();
for (int i=0; i<8; ++i) {
Point estimating_point = center + PointShift2D[i];
if (estimating_point.x < 0
|| estimating_point.x > src.cols-1
|| estimating_point.y < 0
|| estimating_point.y > src.rows-1) {
// estimating_point should not out of the range in image
continue;
} else {
// uchar delta = (uchar)abs(src.at<uchar>(center) - src.at<uchar>(estimating_point));
// delta = (R-R')^2 + (G-G')^2 + (B-B')^2
int delta = int(pow(src.at<Vec3b>(center)[0] - src.at<Vec3b>(estimating_point)[0], 2)
+ pow(src.at<Vec3b>(center)[1] - src.at<Vec3b>(estimating_point)[1], 2)
+ pow(src.at<Vec3b>(center)[2] - src.at<Vec3b>(estimating_point)[2], 2));
if (dest.at<uchar>(estimating_point) == 0
&& mask.at<uchar>(estimating_point) == 0
&& delta < thresh) {
mask.at<uchar>(estimating_point) = 1;
point_stack.push(estimating_point);
}
}
}
}
}
The results are as follow:
To the previous results I apply the following (note that argv1 has different content to the one before, the previous one had the starting image, this next one the result of the previous snippet):
Mat im = imread(argv[1], 0); // the edge image
Mat kernel = getStructuringElement(MORPH_ELLIPSE, Size(11, 11));
Mat morph;
morphologyEx(im, morph, MORPH_CLOSE, kernel);
int rectIdx = 0;
//vector<vector<Point>> contours;
//vector<Vec4i> hierarchy;
findContours(morph, contours, hierarchy, RETR_CCOMP, CHAIN_APPROX_SIMPLE, Point(0, 0));
for (size_t idx = 0; idx < contours.size(); idx++)
{
RotatedRect rect = minAreaRect(contours[idx]);
double areaRatio = abs(contourArea(contours[idx])) / (rect.size.width * rect.size.height);
if (areaRatio > .99)
{
rectIdx = idx;
break;
}
}
//cout << contours.size() << ", rectIdx: " <<rectIdx<<", cont[idx] = "<<contours[rectIdx]<<endl;
// get the convexhull of the contour
vector<Point> hull;
convexHull(contours[rectIdx], hull, false, true);
// visualization
Mat rgb;
Mat only_contours = Mat::zeros(im.rows, im.cols, CV_8UC1);
cvtColor(im, rgb, COLOR_GRAY2RGB);
//drawContours(rgb, contours, rectIdx, Scalar(0, 0, 255), 2);
for(size_t i = 0; i < hull.size(); i++)
{
line(rgb, hull[i], hull[(i + 1)%hull.size()], Scalar(0, 255, 0), 2);
line(only_contours, hull[i], hull[(i + 1)%hull.size()], Scalar(255), 2);
}
imshow("Result", rgb);
imshow("Contour", only_contours);
imwrite("./contour.png", only_contours);
vector<Point> approx;
double d=0;
do
{
d=d+0.5;
approxPolyDP(contours[0],approx,d,true);
//cout << approx.size() << " " <<d<<endl;
}
while (approx.size()>4);
cout << "Approx: " << approx << endl;
contours.push_back(approx);
drawContours(rgb,contours,contours.size()-1,Scalar(0, 0, 255),3);
imshow("Ctr",rgb);
Resulting in the following (in red):
What I would like is for the detected contour of the playing field to get as much of the "white part" of the image as possible, and for cases like the one of the second image to be able to go outside the area delimited by that same white pixels in order to get the best estimate of the playing field's edges. Any ideas on how to do that?
Edit: By changing the parameter that approxPolyDM works on from contours[0] to approx for the iterations after the first, I managed to get slightly better results, still not quite satisfactory though:
P.S. One important thing to note is that the clips given to the first part of the program come from different games where the playing field has no fixed color.