/*M/////////////////////////////////////////////////////////////////////////////////////// // // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. // // By downloading, copying, installing or using the software you agree to this license. // If you do not agree to this license, do not download, install, // copy or use the software. // // // License Agreement // For Open Source Computer Vision Library // // Copyright (C) 2014, OpenCV Foundation, all rights reserved. // Third party copyrights are property of their respective owners. // // Redistribution and use in source and binary forms, with or without modification, // are permitted provided that the following conditions are met: // // * Redistribution's of source code must retain the above copyright notice, // this list of conditions and the following disclaimer. // // * Redistribution's in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimer in the documentation // and/or other materials provided with the distribution. // // * The name of the copyright holders may not be used to endorse or promote products // derived from this software without specific prior written permission. // // This software is provided by the copyright holders and contributors "as is" and // any express or implied warranties, including, but not limited to, the implied // warranties of merchantability and fitness for a particular purpose are disclaimed. // In no event shall the Intel Corporation or contributors be liable for any direct, // indirect, incidental, special, exemplary, or consequential damages // (including, but not limited to, procurement of substitute goods or services; // loss of use, data, or profits; or business interruption) however caused // and on any theory of liability, whether in contract, strict liability, // or tort (including negligence or otherwise) arising in any way out of // the use of this software, even if advised of the possibility of such damage. // //M*/ #ifndef __OPENCV_SALIENCY_SPECIALIZED_CLASSES_HPP__ #define __OPENCV_SALIENCY_SPECIALIZED_CLASSES_HPP__ #include #include #include #include #include "saliencyBaseClasses.hpp" #include "opencv2/core.hpp" namespace cv { namespace saliency { //! @addtogroup saliency //! @{ /************************************ Specific Static Saliency Specialized Classes ************************************/ /** @brief the Spectral Residual approach from @cite SR Starting from the principle of natural image statistics, this method simulate the behavior of pre-attentive visual search. The algorithm analyze the log spectrum of each image and obtain the spectral residual. Then transform the spectral residual to spatial domain to obtain the saliency map, which suggests the positions of proto-objects. */ class CV_EXPORTS_W StaticSaliencySpectralResidual : public StaticSaliency { public: StaticSaliencySpectralResidual(); virtual ~StaticSaliencySpectralResidual(); CV_WRAP static Ptr create() { return makePtr(); } CV_WRAP bool computeSaliency( InputArray image, OutputArray saliencyMap ) { if( image.empty() ) return false; return computeSaliencyImpl( image, saliencyMap ); } CV_WRAP void read( const FileNode& fn ) CV_OVERRIDE; void write( FileStorage& fs ) const CV_OVERRIDE; CV_WRAP int getImageWidth() const { return resImWidth; } CV_WRAP inline void setImageWidth(int val) { resImWidth = val; } CV_WRAP int getImageHeight() const { return resImHeight; } CV_WRAP void setImageHeight(int val) { resImHeight = val; } protected: bool computeSaliencyImpl( InputArray image, OutputArray saliencyMap ) CV_OVERRIDE; CV_PROP_RW int resImWidth; CV_PROP_RW int resImHeight; }; /** @brief the Fine Grained Saliency approach from @cite FGS This method calculates saliency based on center-surround differences. High resolution saliency maps are generated in real time by using integral images. */ class CV_EXPORTS_W StaticSaliencyFineGrained : public StaticSaliency { public: StaticSaliencyFineGrained(); CV_WRAP static Ptr create() { return makePtr(); } CV_WRAP bool computeSaliency( InputArray image, OutputArray saliencyMap ) { if( image.empty() ) return false; return computeSaliencyImpl( image, saliencyMap ); } virtual ~StaticSaliencyFineGrained(); protected: bool computeSaliencyImpl( InputArray image, OutputArray saliencyMap ) CV_OVERRIDE; private: void calcIntensityChannel(Mat src, Mat dst); void copyImage(Mat src, Mat dst); void getIntensityScaled(Mat integralImage, Mat gray, Mat saliencyOn, Mat saliencyOff, int neighborhood); float getMean(Mat srcArg, Point2i PixArg, int neighbourhood, int centerVal); void mixScales(Mat *saliencyOn, Mat intensityOn, Mat *saliencyOff, Mat intensityOff, const int numScales); void mixOnOff(Mat intensityOn, Mat intensityOff, Mat intensity); void getIntensity(Mat srcArg, Mat dstArg, Mat dstOnArg, Mat dstOffArg, bool generateOnOff); }; /************************************ Specific Motion Saliency Specialized Classes ************************************/ /*! * A Fast Self-tuning Background Subtraction Algorithm. * * This background subtraction algorithm is inspired to the work of B. Wang and P. Dudek [2] * [2] B. Wang and P. Dudek "A Fast Self-tuning Background Subtraction Algorithm", in proc of IEEE Workshop on Change Detection, 2014 * */ /** @brief the Fast Self-tuning Background Subtraction Algorithm from @cite BinWangApr2014 */ class CV_EXPORTS_W MotionSaliencyBinWangApr2014 : public MotionSaliency { public: MotionSaliencyBinWangApr2014(); virtual ~MotionSaliencyBinWangApr2014(); CV_WRAP static Ptr create() { return makePtr(); } CV_WRAP bool computeSaliency( InputArray image, OutputArray saliencyMap ) { if( image.empty() ) return false; return computeSaliencyImpl( image, saliencyMap ); } /** @brief This is a utility function that allows to set the correct size (taken from the input image) in the corresponding variables that will be used to size the data structures of the algorithm. @param W width of input image @param H height of input image */ CV_WRAP void setImagesize( int W, int H ); /** @brief This function allows the correct initialization of all data structures that will be used by the algorithm. */ CV_WRAP bool init(); CV_WRAP int getImageWidth() const { return imageWidth; } CV_WRAP inline void setImageWidth(int val) { imageWidth = val; } CV_WRAP int getImageHeight() const { return imageHeight; } CV_WRAP void setImageHeight(int val) { imageHeight = val; } protected: /** @brief Performs all the operations and calls all internal functions necessary for the accomplishment of the Fast Self-tuning Background Subtraction Algorithm algorithm. @param image input image. According to the needs of this specialized algorithm, the param image is a single *Mat*. @param saliencyMap Saliency Map. Is a binarized map that, in accordance with the nature of the algorithm, highlights the moving objects or areas of change in the scene. The saliency map is given by a single *Mat* (one for each frame of an hypothetical video stream). */ bool computeSaliencyImpl( InputArray image, OutputArray saliencyMap ) CV_OVERRIDE; private: // classification (and adaptation) functions bool fullResolutionDetection( const Mat& image, Mat& highResBFMask ); bool lowResolutionDetection( const Mat& image, Mat& lowResBFMask ); // Background model maintenance functions bool templateOrdering(); bool templateReplacement( const Mat& finalBFMask, const Mat& image ); // Decision threshold adaptation and Activity control function bool activityControl(const Mat& current_noisePixelsMask); bool decisionThresholdAdaptation(); // changing structure std::vector > backgroundModel;// The vector represents the background template T0---TK of reference paper. // Matrices are two-channel matrix. In the first layer there are the B (background value) // for each pixel. In the second layer, there are the C (efficacy) value for each pixel Mat potentialBackground;// Two channel Matrix. For each pixel, in the first level there are the Ba value (potential background value) // and in the secon level there are the Ca value, the counter for each potential value. Mat epslonPixelsValue;// epslon threshold Mat activityPixelsValue;// Activity level of each pixel //vector noisePixelMask; // We define a ‘noise-pixel’ as a pixel that has been classified as a foreground pixel during the full resolution Mat noisePixelMask;// We define a ‘noise-pixel’ as a pixel that has been classified as a foreground pixel during the full resolution //detection process,however, after the low resolution detection, it has become a // background pixel. The matrix is two-channel matrix. In the first layer there is the mask ( the identified noise-pixels are set to 1 while other pixels are 0) // for each pixel. In the second layer, there is the value of activity level A for each pixel. //fixed parameter bool activityControlFlag; bool neighborhoodCheck; int N_DS;// Number of template to be downsampled and used in lowResolutionDetection function CV_PROP_RW int imageWidth;// Width of input image CV_PROP_RW int imageHeight;//Height of input image int K;// Number of background model template int N;// NxN is the size of the block for downsampling in the lowlowResolutionDetection float alpha;// Learning rate int L0, L1;// Upper-bound values for C0 and C1 (efficacy of the first two template (matrices) of backgroundModel int thetaL;// T0, T1 swap threshold int thetaA;// Potential background value threshold int gamma;// Parameter that controls the time that the newly updated long-term background value will remain in the // long-term template, regardless of any subsequent background changes. A relatively large (eg gamma=3) will //restrain the generation of ghosts. uchar Ainc;// Activity Incrementation; int Bmax;// Upper-bound value for pixel activity int Bth;// Max activity threshold int Binc, Bdec;// Threshold for pixel-level decision threshold (epslon) adaptation float deltaINC, deltaDEC;// Increment-decrement value for epslon adaptation int epslonMIN, epslonMAX;// Range values for epslon threshold }; /************************************ Specific Objectness Specialized Classes ************************************/ /** * \brief Objectness algorithms based on [3] * [3] Cheng, Ming-Ming, et al. "BING: Binarized normed gradients for objectness estimation at 300fps." IEEE CVPR. 2014. */ /** @brief the Binarized normed gradients algorithm from @cite BING */ class CV_EXPORTS_W ObjectnessBING : public Objectness { public: ObjectnessBING(); virtual ~ObjectnessBING(); CV_WRAP static Ptr create() { return makePtr(); } CV_WRAP bool computeSaliency( InputArray image, OutputArray saliencyMap ) { if( image.empty() ) return false; return computeSaliencyImpl( image, saliencyMap ); } CV_WRAP void read(); CV_WRAP void write() const; /** @brief Return the list of the rectangles' objectness value, in the same order as the *vector\ objectnessBoundingBox* returned by the algorithm (in computeSaliencyImpl function). The bigger value these scores are, it is more likely to be an object window. */ CV_WRAP std::vector getobjectnessValues(); /** @brief This is a utility function that allows to set the correct path from which the algorithm will load the trained model. @param trainingPath trained model path */ CV_WRAP void setTrainingPath( const String& trainingPath ); /** @brief This is a utility function that allows to set an arbitrary path in which the algorithm will save the optional results (ie writing on file the total number and the list of rectangles returned by objectess, one for each row). @param resultsDir results' folder path */ CV_WRAP void setBBResDir( const String& resultsDir ); CV_WRAP double getBase() const { return _base; } CV_WRAP inline void setBase(double val) { _base = val; } CV_WRAP int getNSS() const { return _NSS; } CV_WRAP void setNSS(int val) { _NSS = val; } CV_WRAP int getW() const { return _W; } CV_WRAP void setW(int val) { _W = val; } protected: /** @brief Performs all the operations and calls all internal functions necessary for the accomplishment of the Binarized normed gradients algorithm. @param image input image. According to the needs of this specialized algorithm, the param image is a single *Mat* @param objectnessBoundingBox objectness Bounding Box vector. According to the result given by this specialized algorithm, the objectnessBoundingBox is a *vector\*. Each bounding box is represented by a *Vec4i* for (minX, minY, maxX, maxY). */ bool computeSaliencyImpl( InputArray image, OutputArray objectnessBoundingBox ) CV_OVERRIDE; private: class FilterTIG { public: void update( Mat &w ); // For a W by H gradient magnitude map, find a W-7 by H-7 CV_32F matching score map Mat matchTemplate( const Mat &mag1u ); float dot( int64_t tig1, int64_t tig2, int64_t tig4, int64_t tig8 ); void reconstruct( Mat &w );// For illustration purpose private: static const int NUM_COMP = 2;// Number of components static const int D = 64;// Dimension of TIG int64_t _bTIGs[NUM_COMP];// Binary TIG features float _coeffs1[NUM_COMP];// Coefficients of binary TIG features // For efficiently deals with different bits in CV_8U gradient map float _coeffs2[NUM_COMP], _coeffs4[NUM_COMP], _coeffs8[NUM_COMP]; }; template struct ValStructVec { ValStructVec(); int size() const; void clear(); void reserve( int resSz ); void pushBack( const VT& val, const ST& structVal ); const VT& operator ()( int i ) const; const ST& operator []( int i ) const; VT& operator ()( int i ); ST& operator []( int i ); void sort( bool descendOrder = true ); const std::vector &getSortedStructVal(); std::vector > getvalIdxes(); void append( const ValStructVec &newVals, int startV = 0 ); std::vector structVals; // struct values int sz;// size of the value struct vector std::vector > valIdxes;// Indexes after sort bool smaller() { return true; } std::vector sortedStructVals; }; enum { MAXBGR, HSV, G }; double _base, _logBase; // base for window size quantization int _W;// As described in the paper: #Size, Size(_W, _H) of feature window. int _NSS;// Size for non-maximal suppress int _maxT, _minT, _numT;// The minimal and maximal dimensions of the template int _Clr;// static const char* _clrName[3]; // Names and paths to read model and to store results std::string _modelName, _bbResDir, _trainingPath, _resultsDir; std::vector _svmSzIdxs;// Indexes of active size. It's equal to _svmFilters.size() and _svmReW1f.rows Mat _svmFilter;// Filters learned at stage I, each is a _H by _W CV_32F matrix FilterTIG _tigF;// TIG filter Mat _svmReW1f;// Re-weight parameters learned at stage II. // List of the rectangles' objectness value, in the same order as // the vector objectnessBoundingBox returned by the algorithm (in computeSaliencyImpl function) std::vector objectnessValues; private: // functions inline static float LoG( float x, float y, float delta ) { float d = - ( x * x + y * y ) / ( 2 * delta * delta ); return -1.0f / ( (float) ( CV_PI ) * (delta*delta*delta*delta) ) * ( 1 + d ) * exp( d ); } // Laplacian of Gaussian // Read matrix from binary file static bool matRead( const std::string& filename, Mat& M ); void setColorSpace( int clr = MAXBGR ); // Load trained model. int loadTrainedModel();// Return -1, 0, or 1 if partial, none, or all loaded // Get potential bounding boxes, each of which is represented by a Vec4i for (minX, minY, maxX, maxY). // The trained model should be prepared before calling this function: loadTrainedModel() or trainStageI() + trainStageII(). // Use numDet to control the final number of proposed bounding boxes, and number of per size (scale and aspect ratio) void getObjBndBoxes( Mat &img3u, ValStructVec &valBoxes, int numDetPerSize = 120 ); void getObjBndBoxesForSingleImage( Mat img, ValStructVec &boxes, int numDetPerSize ); bool filtersLoaded() { int n = (int) _svmSzIdxs.size(); return n > 0 && _svmReW1f.size() == Size( 2, n ) && _svmFilter.size() == Size( _W, _W ); } void predictBBoxSI( Mat &mag3u, ValStructVec &valBoxes, std::vector &sz, int NUM_WIN_PSZ = 100, bool fast = true ); void predictBBoxSII( ValStructVec &valBoxes, const std::vector &sz ); // Calculate the image gradient: center option as in VLFeat void gradientMag( Mat &imgBGR3u, Mat &mag1u ); static void gradientRGB( Mat &bgr3u, Mat &mag1u ); static void gradientGray( Mat &bgr3u, Mat &mag1u ); static void gradientHSV( Mat &bgr3u, Mat &mag1u ); static void gradientXY( Mat &x1i, Mat &y1i, Mat &mag1u ); static inline int bgrMaxDist( const Vec3b &u, const Vec3b &v ) { int b = abs( u[0] - v[0] ), g = abs( u[1] - v[1] ), r = abs( u[2] - v[2] ); b = max( b, g ); return max( b, r ); } static inline int vecDist3b( const Vec3b &u, const Vec3b &v ) { return abs( u[0] - v[0] ) + abs( u[1] - v[1] ) + abs( u[2] - v[2] ); } //Non-maximal suppress static void nonMaxSup( Mat &matchCost1f, ValStructVec &matchCost, int NSS = 1, int maxPoint = 50, bool fast = true ); }; //! @} } /* namespace saliency */ } /* namespace cv */ #endif