#include "opencv2/core/utility.hpp" #include "opencv2/highgui.hpp" #include "opencv2/tracking.hpp" #include "opencv2/videoio.hpp" #include "opencv2/plot.hpp" #include "samples_utility.hpp" #include #include #include using namespace std; using namespace cv; // TODO: do normalization ala Kalal's assessment protocol for TLD static const Scalar gtColor = Scalar(0, 255, 0); static Scalar getNextColor() { const int num = 6; static Scalar colors[num] = {Scalar(160, 0, 0), Scalar(0, 0, 160), Scalar(0, 160, 160), Scalar(160, 160, 0), Scalar(160, 0, 160), Scalar(20, 50, 160)}; static int id = 0; return colors[id < num ? id++ : num - 1]; } inline vector readGT(const string &filename, const string &omitname) { vector res; { ifstream input(filename.c_str()); if (!input.is_open()) CV_Error(Error::StsError, "Failed to open file"); while (input) { Rect2d one; input >> one.x; input.ignore(numeric_limits::max(), ','); input >> one.y; input.ignore(numeric_limits::max(), ','); input >> one.width; input.ignore(numeric_limits::max(), ','); input >> one.height; input.ignore(numeric_limits::max(), '\n'); if (input.good()) res.push_back(one); } } if (!omitname.empty()) { ifstream input(omitname.c_str()); if (!input.is_open()) CV_Error(Error::StsError, "Failed to open file"); while (input) { unsigned int a = 0, b = 0; input >> a >> b; input.ignore(numeric_limits::max(), '\n'); if (a > 0 && b > 0 && a < res.size() && b < res.size()) { if (a > b) swap(a, b); for (vector::iterator i = res.begin() + a; i != res.begin() + b; ++i) { *i = Rect2d(); } } } } return res; } inline bool isGoodBox(const Rect2d &box) { return box.width > 0. && box.height > 0.; } const int LTRC_COUNT = 100; struct AlgoWrap { AlgoWrap(const string &name_) : lastState(NotFound), name(name_), color(getNextColor()), numTotal(0), numResponse(0), numPresent(0), numCorrect_0(0), numCorrect_0_5(0), timeTotal(0), auc(LTRC_COUNT + 1, 0) { tracker = createTrackerByName(name); } enum State { NotFound, Overlap_None, Overlap_0, Overlap_0_5, }; Ptr tracker; bool lastRes; Rect2d lastBox; State lastState; // visual string name; Scalar color; // results int numTotal; // frames passed to tracker int numResponse; // frames where tracker had response int numPresent; // frames where ground truth result present int numCorrect_0; // frames where overlap with GT > 0 int numCorrect_0_5; // frames where overlap with GT > 0.5 int64 timeTotal; // ticks vector auc; // number of frames for each overlap percent void eval(const Mat &frame, const Rect2d >Box, bool isVerbose) { // RUN lastBox = Rect2d(); int64 frameTime = getTickCount(); lastRes = tracker->update(frame, lastBox); frameTime = getTickCount() - frameTime; // RESULTS double intersectArea = (gtBox & lastBox).area(); double unionArea = (gtBox | lastBox).area(); numTotal++; numResponse += (lastRes && isGoodBox(lastBox)) ? 1 : 0; numPresent += isGoodBox(gtBox) ? 1 : 0; double overlap = unionArea > 0. ? intersectArea / unionArea : 0.; numCorrect_0 += overlap > 0. ? 1 : 0; numCorrect_0_5 += overlap > 0.5 ? 1 : 0; auc[std::min(std::max((size_t)(overlap * LTRC_COUNT), (size_t)0), (size_t)LTRC_COUNT)]++; timeTotal += frameTime; if (isVerbose) cout << name << " - " << overlap << endl; if (isGoodBox(gtBox) != isGoodBox(lastBox)) lastState = NotFound; else if (overlap > 0.5) lastState = Overlap_0_5; else if (overlap > 0.0001) lastState = Overlap_0; else lastState = Overlap_None; } void draw(Mat &image, const Point &textPoint) const { if (lastRes) rectangle(image, lastBox, color, 2, LINE_8); string suf; switch (lastState) { case AlgoWrap::NotFound: suf = " X"; break; case AlgoWrap::Overlap_None: suf = " ~"; break; case AlgoWrap::Overlap_0: suf = " +"; break; case AlgoWrap::Overlap_0_5: suf = " ++"; break; } putText(image, name + suf, textPoint, FONT_HERSHEY_PLAIN, 1, color, 1, LINE_AA); } // calculates "lost track ratio" curve - row of values growing from 0 to 1 // number of elements is LTRC_COUNT + 2 Mat getLTRC() const { Mat t, res; Mat(auc).convertTo(t, CV_64F); // integral does not support CV_32S input integral(t.t(), res, CV_64F); // t is a column of values return res.row(1) / (double)numTotal; } void plotLTRC(Mat &img) const { Ptr p_ = plot::Plot2d::create(getLTRC()); p_->render(img); } double calcAUC() const { return cv::sum(getLTRC())[0] / (double)LTRC_COUNT; } void stat(ostream &out) const { out << name << endl; out << setw(20) << "Overlap > 0 " << setw(20) << (double)numCorrect_0 / numTotal * 100 << "%" << setw(20) << numCorrect_0 << endl; out << setw(20) << "Overlap > 0.5" << setw(20) << (double)numCorrect_0_5 / numTotal * 100 << "%" << setw(20) << numCorrect_0_5 << endl; double p = (double)numCorrect_0_5 / numResponse; double r = (double)numCorrect_0_5 / numPresent; double f = 2 * p * r / (p + r); out << setw(20) << "Precision" << setw(20) << p * 100 << "%" << endl; out << setw(20) << "Recall " << setw(20) << r * 100 << "%" << endl; out << setw(20) << "f-measure" << setw(20) << f * 100 << "%" << endl; out << setw(20) << "AUC" << setw(20) << calcAUC() << endl; double s = (timeTotal / getTickFrequency()) / numTotal; out << setw(20) << "Performance" << setw(20) << s * 1000 << " ms/frame" << setw(20) << 1 / s << " fps" << endl; } }; inline ostream &operator<<(ostream &out, const AlgoWrap &w) { w.stat(out); return out; } inline vector initAlgorithms(const string &algList) { vector res; istringstream input(algList); for (;;) { char one[30]; input.getline(one, 30, ','); if (!input) break; cout << " " << one << " - "; AlgoWrap a(one); if (a.tracker) { res.push_back(a); cout << "OK"; } else { cout << "FAILED"; } cout << endl; } return res; } static const string &window = "Tracking API"; int main(int argc, char **argv) { const string keys = "{help h||show help}" "{video||video file to process}" "{gt||ground truth file (each line describes rectangle in format: ',,,')}" "{start|0|starting frame}" "{num|0|frame number (0 for all)}" "{omit||file with omit ranges (each line describes occluded frames: ' ')}" "{plot|false|plot LTR curves at the end}" "{v|false|print each frame info}" "{@algos||comma-separated algorithm names}"; CommandLineParser p(argc, argv, keys); if (p.has("help")) { p.printMessage(); return 0; } int startFrame = p.get("start"); int frameCount = p.get("num"); string videoFile = p.get("video"); string gtFile = p.get("gt"); string omitFile = p.get("omit"); string algList = p.get("@algos"); bool doPlot = p.get("plot"); bool isVerbose = p.get("v"); if (!p.check()) { p.printErrors(); return 0; } cout << "Reading GT from " << gtFile << " ... "; vector gt = readGT(gtFile, omitFile); if (gt.empty()) CV_Error(Error::StsError, "Failed to read GT file"); cout << gt.size() << " boxes" << endl; cout << "Opening video " << videoFile << " ... "; VideoCapture cap; cap.open(videoFile); if (!cap.isOpened()) CV_Error(Error::StsError, "Failed to open video file"); cap.set(CAP_PROP_POS_FRAMES, startFrame); cout << "at frame " << startFrame << endl; // INIT vector algos = initAlgorithms(algList); Mat frame, image; cap >> frame; for (vector::iterator i = algos.begin(); i != algos.end(); ++i) i->tracker->init(frame, gt[0]); // DRAW { namedWindow(window, WINDOW_AUTOSIZE); frame.copyTo(image); rectangle(image, gt[0], gtColor, 2, LINE_8); imshow(window, image); } bool paused = false; int frameId = 0; cout << "Hot keys:" << endl << " q - exit" << endl << " p - pause" << endl; for (;;) { if (!paused) { cap >> frame; if (frame.empty()) { cout << "Done - video end" << endl; break; } frameId++; if (isVerbose) cout << endl << "Frame " << frameId << endl; // EVAL for (vector::iterator i = algos.begin(); i != algos.end(); ++i) i->eval(frame, gt[frameId], isVerbose); // DRAW { Point textPoint(1, 16); frame.copyTo(image); rectangle(image, gt[frameId], gtColor, 2, LINE_8); putText(image, "GROUND TRUTH", textPoint, FONT_HERSHEY_PLAIN, 1, gtColor, 1, LINE_AA); for (vector::iterator i = algos.begin(); i != algos.end(); ++i) { textPoint.y += 14; i->draw(image, textPoint); } imshow(window, image); } } char c = (char)waitKey(1); if (c == 'q') { cout << "Done - manual exit" << endl; break; } else if (c == 'p') { paused = !paused; } if (frameCount && frameId >= frameCount) { cout << "Done - max frame count" << endl; break; } } // STAT for (vector::iterator i = algos.begin(); i != algos.end(); ++i) cout << "==========" << endl << *i << endl; if (doPlot) { Mat img(300, 300, CV_8UC3); for (vector::iterator i = algos.begin(); i != algos.end(); ++i) { i->plotLTRC(img); imshow("LTR curve for " + i->name, img); } waitKey(0); } return 0; }